flannel 关闭SNAT

flannel 关闭SNAT

默认情况下，flannel 访问集群外网络是通过 SNAT 成宿主机 ip 方式，在一些金融客户环境中为了能实现防火墙规则，需要直接针对 POD ip 进行进行规则配置，所以需要关闭 SNAT

• 关闭flannel 配置文件关于ip-masq 配置,删除配置文件 -ip-masq 参数

# cat /etc/systemd/system/flanneld.service

[Unit]
Description=Flanneld overlay address etcd agent
After=network.target
After=network-online.target
Wants=network-online.target
After=etcd.service
Before=docker.service

[Service]
Type=notify
ExecStart=/opt/k8s/bin/flanneld \
  -etcd-cafile=/data/work/ca.pem  \
  -etcd-certfile=/data/work/flanneld.pem \
  -etcd-keyfile=/data/work/flanneld-key.pem \
  -etcd-endpoints=https://10.65.91.161:2379,https://10.65.91.162:2379,https://10.65.91.163:2379 \
  -etcd-prefix=/kubernetes/network \
  -iface=ens192
ExecStartPost=/opt/k8s/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker
Restart=always
RestartSec=5
StartLimitInterval=0

[Install]
WantedBy=multi-user.target
RequiredBy=docker.service

#重启 flanneld
systemctl  daemon-reload
systemctl  restart flanneld

• 修改flannel 配置文件中 mk-docker-opts.sh，将 ipmasq=true 修改为 ipmasq=ipmasq=false

#!/bin/sh

usage() {
        echo "$0 [-f FLANNEL-ENV-FILE] [-d DOCKER-ENV-FILE] [-i] [-c] [-m] [-k COMBINED-KEY]

Generate Docker daemon options based on flannel env file
OPTIONS:
        -f      Path to flannel env file. Defaults to /run/flannel/subnet.env
        -d      Path to Docker env file to write to. Defaults to /run/docker_opts.env
        -i      Output each Docker option as individual var. e.g. DOCKER_OPT_MTU=1500
        -c      Output combined Docker options into DOCKER_OPTS var
        -k      Set the combined options key to this value (default DOCKER_OPTS=)
        -m      Do not output --ip-masq (useful for older Docker version)
" >&2

        exit 1
}

flannel_env="/run/flannel/subnet.env"
docker_env="/run/docker_opts.env"
combined_opts_key="DOCKER_OPTS"
indiv_opts=false
combined_opts=false
ipmasq=false
while getopts "f:d:icmk:?h" opt; do
        case $opt in
                f)
                        flannel_env=$OPTARG
                        ;;
                d)
                        docker_env=$OPTARG
                        ;;
                i)
                        indiv_opts=true
                        ;;
                c)
                        combined_opts=true
                        ;;
                m)
                        ipmasq=false
                        ;;
                k)
                        combined_opts_key=$OPTARG
                        ;;
                [\?h])
                        usage
                        ;;
        esac
done

if [ $indiv_opts = false ] && [ $combined_opts = false ]; then
        indiv_opts=true
        combined_opts=true
fi

if [ -f "$flannel_env" ]; then
        . $flannel_env
fi

if [ -n "$FLANNEL_SUBNET" ]; then
        DOCKER_OPT_BIP="--bip=$FLANNEL_SUBNET"
fi

if [ -n "$FLANNEL_MTU" ]; then
        DOCKER_OPT_MTU="--mtu=$FLANNEL_MTU"
fi

if [ -n "$FLANNEL_IPMASQ" ] && [ $ipmasq = false ] ; then
        if [ "$FLANNEL_IPMASQ" = true ] ; then
                DOCKER_OPT_IPMASQ="--ip-masq=false"
        elif [ "$FLANNEL_IPMASQ" = false ] ; then
                DOCKER_OPT_IPMASQ="--ip-masq=false"
        else
                echo "Invalid value of FLANNEL_IPMASQ: $FLANNEL_IPMASQ" >&2
                exit 1
        fi
fi

eval docker_opts="\$${combined_opts_key}"

if [ "$docker_opts" ]; then
        docker_opts="$docker_opts ";
fi

echo -n "" >$docker_env

for opt in $(set | grep "DOCKER_OPT_"); do

        OPT_NAME=$(echo $opt | awk -F "=" '{print $1;}');
        OPT_VALUE=$(eval echo "\$$OPT_NAME");

        if [ "$indiv_opts" = true ]; then
                echo "$OPT_NAME=\"$OPT_VALUE\"" >>$docker_env;
        fi

        docker_opts="$docker_opts $OPT_VALUE";

done

if [ "$combined_opts" = true ]; then
        echo "${combined_opts_key}=\"${docker_opts}\"" >>$docker_env
fi

#重启 flanneld
systemctl  daemon-reload
systemctl  restart flanneld

• docker 配置

# cat /usr/lib/systemd/system/docker.service
#ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock
Environment="PATH=/opt/k8s/bin:/bin:/sbin:/usr/bin:/usr/sbin"
EnvironmentFile=-/run/flannel/docker
ExecStart=/usr/bin/dockerd $DOCKER_NETWORK_OPTIONS

ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always

• 重启docker

systemctl daemon-reload
systemctl restart docker

• 查看docker 配置

#  cat /run/flannel/docker
DOCKER_OPT_BIP="--bip=10.0.79.1/24"
DOCKER_OPT_IPMASQ="--ip-masq=false"
DOCKER_OPT_MTU="--mtu=1500"
DOCKER_NETWORK_OPTIONS=" --bip=10.0.79.1/24 --ip-masq=false --mtu=1500"
[root@lgy-test-node01 10.65.91.164 ~ ]
#  cat /run/flannel/subnet.env
FLANNEL_NETWORK=10.0.0.0/16
FLANNEL_SUBNET=10.0.79.1/24
FLANNEL_MTU=1500
FLANNEL_IPMASQ=false

• 删除 node节点 POSTROUTING 规则,只剩下默认一条规则

iptables -t nat --line-numbers -vnL POSTROUTING

1       87  5856 KUBE-POSTROUTING  all  --  *      *       0.0.0.0/0            0.0.0.0/0            /* kubernetes postrouting rules */

#删除方法
iptables -t nat -D POSTROUTING  3
iptables -t nat -D POSTROUTING  2
iptables -t nat -D POSTROUTING  1

• tcpdump 抓包测试

tcpdump  -i ens192 port 80  -vnn 

10.0.79.2.59010 > 10.65.91.51.80

• 需要增加主机路由

route add -host  10.0.79.2    gw 10.65.91.164