使用netfilter_queue改包笔记

系统：centos 7

准备：安装libnetfilter_queue模块，可以yum安装，也可以网上下载rpm包安装

简介：使用iptables在NAT表上创建DNAT与SNAT规则，对数据包进行转发；在MANGLE表上的FORWARD链上创建NF_QUEUE规则对数据进行勾取并修改；（iptables只有mangle表可以修改数据）

示例规则：

//把到本机 50.24  8889端口的数据包，nat到50.4的8889端口
iptables -t nat -A PREROUTING -p udp -d 192.168.50.24 --dport  -j DNAT --to 192.168.50.4
iptables -t nat -A POSTROUTING -p udp -d 192.168.50.4 --dport  -j SNAT --to 192.168.50.24

//把目的地址50.4，目的端口8889的数据包，入队列 1
iptables -t mangle -A FORWARD -d 192.168.50.4 -p udp --dport   -j NFQUEUE --queue-num 

示例代码：

主线程DoListenIptablesThread负责对QUEUE队列数据的读取，读取到的数据通过回调PacketHandler方法解析处理，传入参数为  queue的 ID号

static void *DoListenIptablesThread(void *pData)
{
    struct nfq_handle *h;
    struct nfq_q_handle *qh;
    struct nfnl_handle *nh;
    int fd;
    int rv;
    int i;
    pthread_t RecvPth[PthNUM];
    char buf[QUEUE_BUFSIZE];
    TCLEANFUNCT struTmp;
    int nTmpError = -;
    int nNum = *(int *)pData;
　　 free(pData);
    pthread_detach(pthread_self());
    memset(&struTmp, , sizeof(struTmp));

    zlog_debug(cat,"opening library handle, nNum[%d]", nNum);

    h = nfq_open();
    if (!h)
    {
        nTmpError = errno;
        zlog_debug(cat,"error during nfq_open(), nNum[%d]", nNum);
        zlog_debug(cat,"nfq_open() errno[%d][%s]", nTmpError, strerror(nTmpError));
        pthread_exit();
    }

    zlog_debug(cat,"unbinding existing nf_queue handler for AF_INET (if any), nNum[%d]", nNum);
    if (nfq_unbind_pf(h, AF_INET) < )
    {
        nTmpError = errno;
        zlog_debug(cat,"error during nfq_unbind_pf(), nNum[%d]", nNum);
        zlog_debug(cat,"nfq_unbind_pf() errno[%d][%s]", nTmpError, strerror(nTmpError));
        nfq_close(h);
        pthread_exit();
    }

    zlog_debug(cat,"binding nfnetlink_queue as nf_queue handler for AF_INET, nNum[%d]", nNum);
    if (nfq_bind_pf(h, AF_INET) < )
    {
        nTmpError = errno;
        zlog_debug(cat,"error during nfq_bind_pf(), nNum[%d]", nNum);
        zlog_debug(cat,"nfq_bind_pf() errno[%d][%s]", nTmpError, strerror(nTmpError));
        nfq_close(h);
        pthread_exit();
    }

    zlog_debug(cat,"binding this socket to queue [%d]", nNum);
    qh = nfq_create_queue(h, nNum, &PacketHandler, &nNum);
    if (!qh)
    {
        nTmpError = errno;
        zlog_debug(cat,"error during nfq_create_queue(), nNum[%d]", nNum);
        zlog_debug(cat,"nfq_create_queue() errno[%d][%s]", nTmpError, strerror(nTmpError));
        nfq_close(h);
        pthread_exit();
    }

    zlog_debug(cat,"setting copy_packet mode, nNum[%d]", nNum);
    if (nfq_set_mode(qh, NFQNL_COPY_PACKET, 0xffff) < )
    {
        nTmpError = errno;
        zlog_debug(cat,"can't set packet_copy mode, nNum[%d]", nNum);
        zlog_debug(cat,"nfq_set_mode() errno[%d][%s]", nTmpError, strerror(nTmpError));
        nfq_destroy_queue(qh);
        nfq_close(h);
        pthread_exit();
    }

    nh = nfq_nfnlh(h);
    fd = nfnl_fd(nh);
    struTmp.qh = qh;
    struTmp.h = h;

    for(i = ;i<PthNUM;i++){
        pthread_create(&RecvPth[i], NULL, DoRecvPacketThread,(void*)&struTmp);
        struTmp.RecvPth[i] = RecvPth[i];
    }
    pthread_cleanup_push(FreePorcessResource, (void*)&struTmp);
    zlog_debug(cat,"Waitting for message ..., nNum[%d]", nNum);
    while ((rv = recv(fd, buf, sizeof(buf), )) && rv >= )
    {
        // 开始处理数据
        //zlog_debug(cat,"-- New packet received -- rv[%d]", rv);
        nfq_handle_packet(h, buf, rv);
　　　　 memset(buf,0x00,sizeof(buf));
    }
    if (rv < ) {
        if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) {
            zlog_debug(cat, "error: [%s], wait for next event.", strerror(errno));
        } else {
            // recv error, free conncetion.
            zlog_error(cat,"recv error: [%s]",strerror(errno));
        }
    }
    pthread_cleanup_pop();
    zlog_error(cat,"-- New packet received -- rv[%d]  fd = [%d]", rv,fd);
    zlog_debug(cat,"Exit DoNetFilter");
}

static int PacketHandler(struct nfq_q_handle *qh, struct nfgenmsg *nfmsg,struct nfq_data *nfa, void *data)
{
    int id = ;
    struct nfqnl_msg_packet_hdr *ph;
    u_int32_t mark,ifi;
    struct iphdr *iph;
    int iphdr_size;
    int ret;char *nf_packet;
    unsigned int nAppProto = -;
    int nReturnValue = ;
    char szHost[] = {};

    ph = nfq_get_msg_packet_hdr(nfa);
    if (ph)
    {
        id = ntohl(ph->packet_id);
    }
    mark = nfq_get_nfmark(nfa);
    if (mark)
    {
        //    DEBUG_LOG("mark=%u ", mark);
    }
    ifi = nfq_get_indev(nfa);
    if (ifi)
    {
        //    DEBUG_LOG("indev=%u ", ifi);
    }
    ifi = nfq_get_outdev(nfa);
    if (ifi)
    {
        //    DEBUG_LOG("outdev=%u ", ifi);
    }
    ret = nfq_get_payload(nfa, (unsigned char**)&nf_packet);
    if ((ret >= ))
    {
        //DEBUG_LOG("payload_len=%d bytes", ret);
        //fputc('\n', stdout);
    }
    // parse the packet headers
    iph = ((struct iphdr *) nf_packet);
    iphdr_size = iph->ihl << ;

    if (iph->protocol == TCP_PRO)
    {
        struct tcphdr *tcp;
        int tcphdr_size;
        int clen;
        tcp = ((struct tcphdr *) (nf_packet + (iph->ihl << )));

        tcphdr_size = (tcp->doff << );
        clen = ret - iphdr_size - tcphdr_size;
        if(clen > 0)
        {
　　　　　//在此处修改数据包，修改数据包后执行下面两行代码，重新对数据进行校验，然后通知内核放行修改后的数据包
        //set_tcp_checksum1(iph);
　　　　　//return nfq_set_verdict(qh, id, NF_ACCEPT,(u_int32_t)ret, nf_packet);
        }
    }
    // if protocol is udp

    if(iph->protocol == UDP_PRO)
    {
        int clen;
        struct udphdr *udp;
        udp = ((struct udphdr *) (nf_packet + (iph->ihl << )));
        clen = ret - iphdr_size - UDP_HEADER_LEN;
        if(clen > )
        {
            char* c;
            PACKETINFO  packinfo;
            memset(&packinfo,0x00, sizeof(struct PACKETINFO));
            c = nf_packet + iphdr_size + UDP_HEADER_LEN;
　　　　　　

            Length_dif = strlen(c) -clen;
            zlog_debug(cat,"[UDP]Length_dif===> %d  clen ==>[%d]",Length_dif,clen);
            iph->tot_len = htons(ntohs(iph->tot_len)+Length_dif);
            iph->check = 0;
            iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);

　　　　　　　//在此处修改数据包，修改数据包后执行下面两行代码，重新对数据进行校验，然后通知内核放行修改后的数据包
            //set_udp_checksum1(iph);
            //return nfq_set_verdict(qh, id, NF_ACCEPT,(u_int32_t)ret, nf_packet);
        }
    }
    return nfq_set_verdict(qh, id, NF_ACCEPT,, NULL);
}

线程退出时资源释放代码：
主线程DoListenIptablesThread中recv的行为为阻塞，所以强制通过其他方式强制退出时，无法有效关闭并释放资源，通过FreePorcessResource对其资源进行关闭回收，并杀掉其开辟的线程；

void FreePorcessResource(void *pData)
{
    TCLEANFUNCT *pTmp = NULL;
    int i;
    int kill_rc;

    pTmp = (TCLEANFUNCT *)pData;
    for(i = ;i<PthNUM;i++){
        if(!pTmp->RecvPth[i]) continue;
        kill_rc = pthread_kill(pTmp->RecvPth[i], );
        if (kill_rc == ESRCH)
        {
            zlog_debug(cat,"the specified thread did not exists or already quit --- ");
        }
        else if (kill_rc == EINVAL)
        {
            zlog_debug(cat,"signal is invalid --- ");
        }
        else
        {
            zlog_debug(cat,"the specified thread is alive --- ");
            // 杀死该线程
            pthread_cancel(pTmp->RecvPth[i]);
            //pthread_join(m->second, NULL);
            usleep(*);

            // 检测该线程是否存在
            kill_rc = pthread_kill(pTmp->RecvPth[i], );
            if (kill_rc == ESRCH)
            {
                zlog_debug(cat,"the specified thread did not exists or already quit +++ ");
            }
            else if (kill_rc == EINVAL)
            {
                zlog_debug(cat,"signal is invalid +++ ");
            }
            else
            {
                zlog_debug(cat,"signal is alive +++ ");
            }
        }
    }

    nfq_destroy_queue(pTmp->qh);
    nfq_close(pTmp->h);
    zlog_debug(cat,"closing pthread handle\n");
}

主线程DoListenIptablesThread创建的数据读取线程：(多核设备时，内核会通过多核接收数据，单线程recv数据时，系统接收缓存区会由于应用层recv过慢造成缓存区没有足够的空间，所以该处需要多线程recv处理)

static void *DoRecvPacketThread(void *pData){
    TCLEANFUNCT *pTmp = NULL;
    int rv;
    int fd;
    char buf[QUEUE_BUFSIZE];
    struct nfnl_handle *nh;
    pthread_detach(pthread_self());
    pTmp = (TCLEANFUNCT *)pData;
    nh = nfq_nfnlh(pTmp->h);
    fd = nfnl_fd(nh);

    while ((rv = recv(fd, buf, sizeof(buf), )) && rv >= )
    {
        // 开始处理数据
        //zlog_debug(cat,"-- New packet received -- rv[%d]", rv);
        nfq_handle_packet(pTmp->h, buf, rv);
　　　　 memset(buf,0x00,sizeof(buf));
    }
    if (rv < ) {
        if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) {
            zlog_debug(cat, "error: [%s], wait for next event.", strerror(errno));
        } else {
            // recv error, free conncetion.
            zlog_error(cat,"recv error: [%s]",strerror(errno));
        }
    }
    zlog_error(cat,"-- New packet received -- rv[%d]  fd = [%d]", rv,fd);
}

TCP与UDP数据修改后重新校验实现：

static u_int16_t checksum(u_int32_t init, u_int8_t *addr, size_t count){
    /* Compute Internet Checksum for "count" bytes * beginning at location "addr". */
    u_int32_t sum = init; 

    while( count >  ) {
        /* This is the inner loop */
        sum += ntohs(* (u_int16_t*) addr);
        addr += ;
        count -= ;
    } /* Add left-over byte, if any */
    if( count >  )
        sum += ntohs(* (u_int8_t*) addr); /* Fold 32-bit sum to 16 bits */
    while (sum>>)
    sum = (sum & 0xffff) + (sum >> );
    return (u_int16_t)~sum;
}
static u_int16_t tcp_checksum2(struct iphdr* iphdrp, struct tcphdr* tcphdrp){
    size_t tcplen = ntohs(iphdrp->tot_len) - (iphdrp->ihl<<);
    u_int32_t cksum = ;

    cksum += ntohs((iphdrp->saddr >> ) & 0x0000ffff);
    cksum += ntohs(iphdrp->saddr & 0x0000ffff);
    cksum += ntohs((iphdrp->daddr >> ) & 0x0000ffff);
    cksum += ntohs(iphdrp->daddr & 0x0000ffff);
    cksum += iphdrp->protocol & 0x00ff;
    cksum += tcplen;
    return checksum(cksum, (u_int8_t*)tcphdrp, tcplen);
} 

static u_int16_t tcp_checksum1(struct iphdr* iphdrp){
    struct tcphdr *tcphdrp = (struct tcphdr*)((u_int8_t*)iphdrp + (iphdrp->ihl<<));
    return tcp_checksum2(iphdrp, tcphdrp);
}
static void set_tcp_checksum2(struct iphdr* iphdrp, struct tcphdr* tcphdrp){
    tcphdrp->check = ;
    tcphdrp->check = htons(tcp_checksum2(iphdrp, tcphdrp));
}
static void set_tcp_checksum1(struct iphdr* iphdrp){
    struct tcphdr *tcphdrp = (struct tcphdr*)((u_int8_t*)iphdrp + (iphdrp->ihl<<));
    set_tcp_checksum2(iphdrp, tcphdrp);
}

static u_int16_t udp_checksum2(struct iphdr* iphdrp, struct udphdr* udphdrp){
    size_t udplen = ntohs(iphdrp->tot_len) - (iphdrp->ihl<<);
    u_int32_t cksum = ;

    cksum += ntohs((iphdrp->saddr >> ) & 0x0000ffff);
    cksum += ntohs(iphdrp->saddr & 0x0000ffff);
    cksum += ntohs((iphdrp->daddr >> ) & 0x0000ffff);
    cksum += ntohs(iphdrp->daddr & 0x0000ffff);
    cksum += iphdrp->protocol & 0x00ff;
    cksum += udplen;
    return checksum(cksum, (u_int8_t*)udphdrp, udplen);
} 

static u_int16_t udp_checksum1(struct iphdr* iphdrp){
    struct udphdr *udphdrp = (struct udphdr*)((u_int8_t*)iphdrp + (iphdrp->ihl<<));
    return udp_checksum2(iphdrp, udphdrp);
}
static void set_udp_checksum2(struct iphdr* iphdrp, struct udphdr* udphdrp){
    udphdrp->check = ;
    udphdrp->check = htons(udp_checksum2(iphdrp, udphdrp));
}
static void set_udp_checksum1(struct iphdr* iphdrp){
    struct udphdr *udphdrp = (struct udphdr*)((u_int8_t*)iphdrp + (iphdrp->ihl<<));
    set_udp_checksum2(iphdrp, udphdrp);
}

ip头部校验

static inline unsigned short ip_fast_csum(unsigned char* iph,unsigned int ihl){
    unsigned int sum;

__asm__ __volatile__(
    "movl (%1), %0 ;\n"
    "subl $4, %2 ;\n"
    "jbe 2f ;\n"
    "addl 4(%1), %0 ;\n"
    "adcl 8(%1), %0 ;\n"
    "adcl 12(%1), %0 ;\n"
    "1: adcl 16(%1), %0 ;\n"
    "lea 4(%1), %1 ;\n"
    "decl %2 ;\n"
    "jne 1b ;\n"
    "adcl $0, %0 ;\n"
    "movl %0, %2 ;\n"   //保存sum的值到%2
    "shrl $16, %0 ;\n"  //右移16位（读取高16位）到%0
    "addw %w2, %w0 ;\n" //%0的16位加%2的16位
    "adcl $0, %0 ;\n"   //若进位加上进位
    "notl %0 ;\n" //取反
    "2: ;\n"
/* Since the input registers which are loaded with iph and ihl
are modified, we must also specify them as outputs, or gcc
will assume they contain their original values. */
    : "=r" (sum), "=r" (iph), "=r" (ihl)
    : "" (iph), "" (ihl)
    : "memory");
    return (sum);
}

程序内部宏定义整理：

#define        MAC_LEN        12
#define        UDP_PRO        17
#define        TCP_PRO        6
#define        VXLAN_HEADER_LEN    8
#define        UDP_HEADER_LEN    8
#define        TCP_HEADER_NO_OPERATION_LEN        20
#define     QUEUE_BUFSIZE    8192
#define     PthNUM            10

typedef struct __CleanFunct
{
    struct nfq_q_handle *qh;
    struct nfq_handle *h;
    pthread_t RecvPth[PthNUM];
}TCLEANFUNCT;

程序需要头文件：

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <linux/ip.h>
#include <linux/tcp.h>
#include <linux/udp.h>
#include <netinet/in.h>
#include <linux/types.h>
#include <pthread.h>
#include <zlog.h>
#include <assert.h>
#include <linux/netfilter.h>
#include <libnetfilter_queue/libnetfilter_queue.h>

代码编译需要链接内容：

-lpthread  -lnfnetlink -lnetfilter_queue

整理不易，转载请注明出处；