DPDK l2fwd 浅注

l2fwd是DPDK中的非常经典的例子。二层转发模型。

就是在相邻的网卡接口间互相传递报文。

网口0和网口1之间报文互传。

网口2和网口3之间报文互传。

。。。。。。。。。。。。

运行参数 .

在目录/home/yml/dpdk/dpdk-stable-16.07.2/examples/l2fwd/build 下面（当然要先编译这个例子 make）

1. ./l2fwd [EAL options]---p PORTMASK [-q NQ]--[no-]mac-updating

EAL options : 就是EAL的参数 ,我这里设成

1. -c 3-n 2

1. -c 表示的是整个程序会使用哪些核
2. 例如：-c f 表示 20+21+22+23
3. 表示 四个核都会用上。

 

-n 表示内存的通道数。

 

-- -p  ： 表示使用哪些网口， 

1. ---p 3

表示使用网口0和1 。

1. -q 1

表示每个逻辑核（lcore线程）上分配一个网口。

 

我这里直接用下面的参数运行。

1. ./l2fwd -c3 -n2 ---p 3-q 1

 

---

 

1. int
2. main(int argc,char**argv)
3. {
4. struct lcore_queue_conf *qconf;
5. struct rte_eth_dev_info dev_info;
6. int ret;
7. uint8_t nb_ports;
8. uint8_t nb_ports_available;
9. uint8_t portid, last_port;
10. unsigned lcore_id, rx_lcore_id;
11. unsigned nb_ports_in_mask =0;
13. /* init EAL */
14. ret = rte_eal_init(argc, argv); //EAL的初使化。
15. if(ret <0)
16. rte_exit(EXIT_FAILURE,"Invalid EAL arguments\n");
17. argc -= ret;
18. argv += ret;
20. force_quit =false;
21. signal(SIGINT, signal_handler);
22. signal(SIGTERM, signal_handler);
24. /* parse application arguments (after the EAL ones) */
25. ret = l2fwd_parse_args(argc, argv);//这里是解析后面的---p 3-q 1 参数，用于获得这些参数的值。
26. if(ret <0)
27. rte_exit(EXIT_FAILURE,"Invalid L2FWD arguments\n");
29. /* convert to number of cycles */
30. timer_period *= rte_get_timer_hz();
32. /* create the mbuf pool */
33. l2fwd_pktmbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", NB_MBUF,
34. MEMPOOL_CACHE_SIZE,0, RTE_MBUF_DEFAULT_BUF_SIZE, //创建内存池，
35. rte_socket_id());
36. if(l2fwd_pktmbuf_pool == NULL)
37. rte_exit(EXIT_FAILURE,"Cannot init mbuf pool\n");
39. nb_ports = rte_eth_dev_count(); //获得当初可用的最大网口数，是从PCI中获得的。
40. if(nb_ports ==0)
41. rte_exit(EXIT_FAILURE,"No Ethernet ports - bye\n");
43. /* reset l2fwd_dst_ports */
44. for(portid =0; portid < RTE_MAX_ETHPORTS; portid++)
45. l2fwd_dst_ports[portid]=0;
46. last_port =0;
48. /*
49. * Each logical core is assigned a dedicated TX queue on each port.
50. */
51. for(portid =0; portid < nb_ports; portid++){ //这里其实就是确定每个网口的发包对象。如网口0的报文传给网口1，。。。。这里就是初使化一个数组。
52. /* skip ports that are not enabled */
53. if((l2fwd_enabled_port_mask &(1<< portid))==0)
54. continue;
56. if(nb_ports_in_mask %2){
57. l2fwd_dst_ports[portid]= last_port;
58. l2fwd_dst_ports[last_port]= portid;
59. }
60. else
61. last_port = portid;
63. nb_ports_in_mask++;
65. rte_eth_dev_info_get(portid,&dev_info);
66. }
67. if(nb_ports_in_mask %2){
68. printf("Notice: odd number of ports in portmask.\n");
69. l2fwd_dst_ports[last_port]= last_port;
70. }
72. rx_lcore_id =0;
73. qconf = NULL;
75. /* Initialize the port/queue configuration of each logical core */
76. for(portid =0; portid < nb_ports; portid++){//给每一个网口分配一个可用的，你配置过的逻辑核
77. /* skip ports that are not enabled */
78. if((l2fwd_enabled_port_mask &(1<< portid))==0)
79. continue;
81. /* get the lcore_id for this port */
82. while(rte_lcore_is_enabled(rx_lcore_id)==0||
83. lcore_queue_conf[rx_lcore_id].n_rx_port ==
84. l2fwd_rx_queue_per_lcore){
85. rx_lcore_id++;
86. if(rx_lcore_id >= RTE_MAX_LCORE)
87. rte_exit(EXIT_FAILURE,"Not enough cores\n");
88. }
90. if(qconf !=&lcore_queue_conf[rx_lcore_id])
91. /* Assigned a new logical core in the loop above. */
92. qconf =&lcore_queue_conf[rx_lcore_id];
94. qconf->rx_port_list[qconf->n_rx_port]= portid;
95. qconf->n_rx_port++;
96. printf("Lcore %u: RX port %u\n", rx_lcore_id,(unsigned) portid);
97. }
99. nb_ports_available = nb_ports;
101. /* Initialise each port */
102. for(portid =0; portid < nb_ports; portid++){//就是初使化网口，给每个网口分配接收的缓存，分配接收和发送的队列。
103. /* skip ports that are not enabled */
104. if((l2fwd_enabled_port_mask &(1<< portid))==0){
105. printf("Skipping disabled port %u\n",(unsigned) portid);
106. nb_ports_available--;
107. continue;
108. }
109. /* init port */
110. printf("Initializing port %u... ",(unsigned) portid);
111. fflush(stdout);
112. ret = rte_eth_dev_configure(portid,1,1,&port_conf);//设置portid这个网口一个接收和一个发送的队列。
113. if(ret <0)
114. rte_exit(EXIT_FAILURE,"Cannot configure device: err=%d, port=%u\n",
115. ret,(unsigned) portid);
117. rte_eth_macaddr_get(portid,&l2fwd_ports_eth_addr[portid]);
119. /* init one RX queue */
120. fflush(stdout);
121. ret = rte_eth_rx_queue_setup(portid,0, nb_rxd, //给对应的网口分配接收队列。
122. rte_eth_dev_socket_id(portid),
123. NULL,
124. l2fwd_pktmbuf_pool);
125. if(ret <0)
126. rte_exit(EXIT_FAILURE,"rte_eth_rx_queue_setup:err=%d, port=%u\n",
127. ret,(unsigned) portid);
129. /* init one TX queue on each port */
130. fflush(stdout);
131. ret = rte_eth_tx_queue_setup(portid,0, nb_txd,//给对应的网口分配发送队列。
132. rte_eth_dev_socket_id(portid),
133. NULL);
134. if(ret <0)
135. rte_exit(EXIT_FAILURE,"rte_eth_tx_queue_setup:err=%d, port=%u\n",
136. ret,(unsigned) portid);
138. /* Initialize TX buffers */
139. tx_buffer[portid]= rte_zmalloc_socket("tx_buffer",//分配接收缓存空间，就是收包的缓存。
140. RTE_ETH_TX_BUFFER_SIZE(MAX_PKT_BURST),0,
141. rte_eth_dev_socket_id(portid));
142. if(tx_buffer[portid]== NULL)
143. rte_exit(EXIT_FAILURE,"Cannot allocate buffer for tx on port %u\n",
144. (unsigned) portid);
146. rte_eth_tx_buffer_init(tx_buffer[portid], MAX_PKT_BURST);
148. ret = rte_eth_tx_buffer_set_err_callback(tx_buffer[portid],//当报文发送失败后要调用的回调函数
149. rte_eth_tx_buffer_count_callback,
150. &port_statistics[portid].dropped);
151. if(ret <0)
152. rte_exit(EXIT_FAILURE,"Cannot set error callback for "
153. "tx buffer on port %u\n",(unsigned) portid);
155. /* Start device */
156. ret = rte_eth_dev_start(portid);
157. if(ret <0)
158. rte_exit(EXIT_FAILURE,"rte_eth_dev_start:err=%d, port=%u\n",
159. ret,(unsigned) portid);
161. printf("done: \n");
163. rte_eth_promiscuous_enable(portid);//设置该网口为混杂模式，就是接收所有的报文。
165. printf("Port %u, MAC address: %02X:%02X:%02X:%02X:%02X:%02X\n\n",//初使化报文统计
166. (unsigned) portid,
167. l2fwd_ports_eth_addr[portid].addr_bytes[0],
168. l2fwd_ports_eth_addr[portid].addr_bytes[1],
169. l2fwd_ports_eth_addr[portid].addr_bytes[2],
170. l2fwd_ports_eth_addr[portid].addr_bytes[3],
171. l2fwd_ports_eth_addr[portid].addr_bytes[4],
172. l2fwd_ports_eth_addr[portid].addr_bytes[5]);
174. /* initialize port stats */
175. memset(&port_statistics,0,sizeof(port_statistics));
176. }
178. if(!nb_ports_available){
179. rte_exit(EXIT_FAILURE,
180. "All available ports are disabled. Please set portmask.\n");
181. }
183. check_all_ports_link_status(nb_ports, l2fwd_enabled_port_mask);//检查这些网口是否可以用。
185. ret =0;
186. /* launch per-lcore init on every lcore */
187. rte_eal_mp_remote_launch(l2fwd_launch_one_lcore, NULL, CALL_MASTER);//在所有的逻辑核(包括管理核，一般DPDK会默认将第一个可用的逻辑核当管理核)上执行l2fwd_launch_one_lcore这个函数。
188. RTE_LCORE_FOREACH_SLAVE(lcore_id){
189. if(rte_eal_wait_lcore(lcore_id)<0){//等待从逻辑核的结束。
190. ret =-1;
191. break;
192. }
193. }
195. for(portid =0; portid < nb_ports; portid++){ //down掉网口，结束。
196. if((l2fwd_enabled_port_mask &(1<< portid))==0)
197. continue;
198. printf("Closing port %d...", portid);
199. rte_eth_dev_stop(portid);
200. rte_eth_dev_close(portid);
201. printf(" Done\n");
202. }
203. printf("Bye...\n");
205. return ret;
206. }

 

1. staticvoid
2. l2fwd_main_loop(void)
3. {
4. struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
5. struct rte_mbuf *m;
6. int sent;
7. unsigned lcore_id;
8. uint64_t prev_tsc, diff_tsc, cur_tsc, timer_tsc;
9. unsigned i, j, portid, nb_rx;
10. struct lcore_queue_conf *qconf;
11. constuint64_t drain_tsc =(rte_get_tsc_hz()+ US_PER_S -1)/ US_PER_S *
12. BURST_TX_DRAIN_US;
13. struct rte_eth_dev_tx_buffer *buffer;
15. prev_tsc =0;
16. timer_tsc =0;
18. lcore_id = rte_lcore_id();
19. qconf =&lcore_queue_conf[lcore_id];//获得我们初使化的时候配置的逻辑核与网口的对应关系列表。
21. if(qconf->n_rx_port ==0){
22. RTE_LOG(INFO, L2FWD,"lcore %u has nothing to do\n", lcore_id);
23. return;
24. }
26. RTE_LOG(INFO, L2FWD,"entering main loop on lcore %u\n", lcore_id);
28. for(i =0; i < qconf->n_rx_port; i++){
30. portid = qconf->rx_port_list[i];
31. RTE_LOG(INFO, L2FWD," -- lcoreid=%u portid=%u\n", lcore_id,
32. portid);
34. }
36. while(!force_quit){
38. cur_tsc = rte_rdtsc();
40. /*
41. * TX burst queue drain
42. */
43. diff_tsc = cur_tsc - prev_tsc;
44. if(unlikely(diff_tsc > drain_tsc)){//时间时隔到了就刷新发送所有网口上的待发送的报文。
46. for(i =0; i < qconf->n_rx_port; i++){
48. portid = l2fwd_dst_ports[qconf->rx_port_list[i]];
49. buffer = tx_buffer[portid];
51. sent = rte_eth_tx_buffer_flush(portid,0, buffer);
52. if(sent)
53. port_statistics[portid].tx += sent;
55. }
57. /* if timer is enabled */
58. if(timer_period >0){
60. /* advance the timer */
61. timer_tsc += diff_tsc;
63. /* if timer has reached its timeout */
64. if(unlikely(timer_tsc >= timer_period)){
66. /* do this only on master core */
67. if(lcore_id == rte_get_master_lcore()){//如果是在逻辑核上，且符合条件，就打印统计信息。
68. print_stats();
69. /* reset the timer */
70. timer_tsc =0;
71. }
72. }
73. }
75. prev_tsc = cur_tsc;
76. }
78. /*
79. * Read packet from RX queues
80. */
81. for(i =0; i < qconf->n_rx_port; i++){//轮询每个网口，进行接收报文。
83. portid = qconf->rx_port_list[i];
84. nb_rx = rte_eth_rx_burst((uint8_t) portid,0,
85. pkts_burst, MAX_PKT_BURST);
87. port_statistics[portid].rx += nb_rx;
89. for(j =0; j < nb_rx; j++){
90. m = pkts_burst[j];
91. rte_prefetch0(rte_pktmbuf_mtod(m,void*));
92. l2fwd_simple_forward(m, portid);
93. }
94. }
95. }
96. }

1. staticvoid
2. l2fwd_simple_forward(struct rte_mbuf *m,unsigned portid)
3. {
4. struct ether_hdr *eth;
5. void*tmp;
6. unsigned dst_port;
7. int sent;
8. struct rte_eth_dev_tx_buffer *buffer;
10. dst_port = l2fwd_dst_ports[portid];//获得发送的网口，相当于就是做转发。
11. eth = rte_pktmbuf_mtod(m,struct ether_hdr *);
13. /* 02:00:00:00:00:xx */
14. tmp =&eth->d_addr.addr_bytes[0];
15. *((uint64_t*)tmp)=0x000000000002+((uint64_t)dst_port <<40);
17. /* src addr */
18. ether_addr_copy(&l2fwd_ports_eth_addr[dst_port],&eth->s_addr);
20. buffer = tx_buffer[dst_port];//获得初使化的时候配置的发送缓存。
21. sent = rte_eth_tx_buffer(dst_port,0, buffer, m);//预发送，其实并没有真正的发送。
22. if(sent)
23. port_statistics[dst_port].tx += sent;
24. }

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

null