LVS之DR跨网段实战及高可用性

author：JevonWei

版权声明：原创作品

---

LVS-DR实现跨网段

网络拓扑



网络环境

RS1
    RIP 192.168.198.138/24
    VIP 192.168.80.100/32
    GW  192.168.198.130
RS2
    RIP 192.168.198.132/24
    VIP 192.168.80.100/32
    GW  192.168.198.130
VS
    DIP 192.168.198.128/24
    VIP 192.168.198.100/32
    GW  192.168.198.130

route
    192.168.198.130/24
    192.168.80.130/8
    172.16.253.166/16
    GW  192.168.198.130
Client
    172.16.254.150/16
    GW 172.16.253.166
RS1,RS2的网关指向192.168.198.130

route

ens38网卡添加第二个IP
[root@route network-scripts]# nmcli connection modify ens38 +ipv4.addresses 192.168.80.130/8
[root@route ~]# nmcli connection up ens38  \\启动ens38网卡
[root@route ~]# ip a
[root@route ~]# route add default gw 192.168.198.130

VS

编辑LVS_DR的配置脚本
[root@VS ~]# vim lvs_dr.sh
#! /bin/bash
vip=192.168.80.100
server=$vip:80
rip1=192.168.198.138
rip2=192.168.198.132
sch=rr
dev=ens34:1
case $1 in
start)
    ifconfig $dev $vip/32 broadcast $vip
    ipvsadm -A -t $server -s $sch
    ipvsadm -a -t $server -r $rip1 -g -w 3
    ipvsadm -a -t $server -r $rip2 -g -w 1
    ;;
stop)
    ipvsadm -C
    ifconfig $dev down
    ;;
*)
    echo "Usage:$(basename $0) start|stop"
    exit 1
    ;;
esac

添加网关及默认路由
[root@VS ~]# route add default gw 192.168.198.130
[root@VS ~]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.198.130 0.0.0.0         UG    0      0        0 ens34
192.168.122.0   0.0.0.0         255.255.255.0   U     0      0        0 virbr0
192.168.198.0   0.0.0.0         255.255.255.0   U     100    0        0 ens34

RS1和RS2配置vip IP

[root@RS1 ~]# vim dr_vip_rs.sh
#!/bin/bash
#
vip=192.168.80.100
mask='255.255.255.255'
dev=lo:1
case $1 in
start)
   echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
   echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
   echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
   echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
   ifconfig $dev $vip netmask $mask broadcast $vip up
   # route add -host $vip dev $dev
   echo "VS server is Ready "
   ;;
stop)
   ifconfig $dev down
   echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
   echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
   echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
   echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
   echo "VS server is Cancel"
   ;;
*)
   echo "Usage $(basename $0) start|stop"
   exit 1
   ;;
esac

[root@RS1 ~]# bash dr_vip_rs.sh start
VS server is Ready
[root@RS2 ~]#  bash dr_vip_rs.sh start
VS server is Ready 

路由信息
[root@RS2 ~]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.198.130 0.0.0.0         UG    100    0        0 ens34
192.168.198.0   0.0.0.0         255.255.255.0   U     100    0        0 ens34
[root@RS1 ~]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
192.168.198.0   0.0.0.0         255.255.255.0   U     0      0        0 eth1
0.0.0.0         192.168.198.130 0.0.0.0         UG    0      0        0 eth1

client

[root@client ~]# for i in {1..10};do curl 192.168.80.100 ;done
welcome to RS2
welcome to RS1
welcome to RS2
welcome to RS1
welcome to RS2
welcome to RS1
welcome to RS2
welcome to RS1
welcome to RS2
welcome to RS1

将http和https两个不同的服务打标签,从而使http和https做成一个集群服务

• FireWall Mark技术

VS

[root@VS ~]# iptables -t mangle -A PREROUTING -d 192.168.80.100 -p tcp -m multiport --dports 80,443 -j MARK --set-mark 10
[root@VS ~]# vim lvs_dr_vs_fwm.sh
#! /bin/bash
vip=192.168.80.100
server=10
rip1=192.168.198.138
rip2=192.168.198.132
sch=rr
dev=ens34:1
case $1 in
start)
    ifconfig $dev $vip/32 broadcast $vip
    ipvsadm -A -f $server -s $sch
    ipvsadm -a -f $server -r $rip1 -g -w 3
    ipvsadm -a -f $server -r $rip2 -g -w 1
    ;;
stop)
    ipvsadm -C
    ifconfig $dev down
    ;;
*)
    echo "Usage:$(basename $0) start|stop"
    exit 1
    ;;
esac

[root@VS ~]# bash lvs_dr_vs_fwm.sh start
[root@VS ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
  FWM  10 rr
  -> 192.168.198.132:0            Route   1      0          0
  -> 192.168.198.138:0            Route   3      0          0

client

[root@client ~]# curl 192.168.80.100；curl -k https://192.168.80.100

实现DR持久连接

PFWMC基于防火墙的持久连接

VS

[root@VS ~]# iptables -t mangle -A PREROUTING -d 192.168.80.100 -p tcp -m multiport --dports 80,443 -j MARK --set-mark 10
[root@VS ~]# vim lvs_dr_vs_fwm.sh
#! /bin/bash
vip=192.168.80.100
server=10
rip1=192.168.198.138
rip2=192.168.198.132
sch=rr
dev=ens34:1
case $1 in
start)
    ifconfig $dev $vip/32 broadcast $vip
    ipvsadm -A -f $server -s $sch -p 600  \\-p 设置持久连接为600s
    ipvsadm -a -f $server -r $rip1 -g -w 3
    ipvsadm -a -f $server -r $rip2 -g -w 1
    ;;
stop)
    ipvsadm -C
    ifconfig $dev down
    ;;
*)
    echo "Usage:$(basename $0) start|stop"
    exit 1
    ;;
esac

[root@VS ~]# bash lvs_dr_vs_fwm.sh start
[root@VS ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
  FWM  10 rr persistent 600 \\持久连接为600s
  -> 192.168.198.132:0            Route   1      0          0
  -> 192.168.198.138:0            Route   3      0          0

client

[root@client ~]# curl 192.168.80.100
welcome to RS2
[root@client ~]# curl 192.168.80.100
welcome to RS2
[root@client ~]# curl 192.168.80.100
welcome to RS2
[root@client ~]# curl https://192.168.80.100
welcome to RS2
[root@client ~]# curl https://192.168.80.100
welcome to RS2

PCC基于0端口的持久连接

VS

[root@VS ~]# iptables -t mangle -A PREROUTING -d 192.168.80.100 -p tcp -m multiport --dports 80,443 -j MARK --set-mark 10
[root@VS ~]# vim lvs_dr_vs_per.sh
#! /bin/bash
vip=192.168.80.100
server=$vip:0
rip1=192.168.198.138
rip2=192.168.198.132
sch=rr
dev=ens34:1
case $1 in
start)
    ifconfig $dev $vip netmask 255.255.255.255 broadcast $vip
    ipvsadm -A -t $server -s $sch -p 600
    ipvsadm -a -t $server -r $rip1 -g -w 3
    ipvsadm -a -t $server -r $rip2 -g -w 1
    ;;
stop)
    ipvsadm -C
    ifconfig $dev down
    ;;
*)
    echo "Usage:$(basename $0) start|stop"
    exit 1
    ;;
esac

[root@VS ~]# bash lvs_dr_vs_per.sh start
[root@VS ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.80.100:0 rr persistent 600
  -> 192.168.198.132:0            Route   1      0          0
  -> 192.168.198.138:0            Route   3      0          0

LVS高可用性

VS(编写脚本判断RS服务器是否故障)

[root@VS ~]# vim lvs_dr_vs.sh
#! /bin/bash
vip=192.168.80.100
server=$vip:80
rip1=192.168.198.138
rip2=192.168.198.132
sch=rr
dev=ens34:1
case $1 in
start)
    ifconfig $dev $vip/32 broadcast $vip
    ipvsadm -A -t $server -s $sch
    ipvsadm -a -t $server -r $rip1 -g -w 3
    ipvsadm -a -t $server -r $rip2 -g -w 1
    ;;
stop)
    ipvsadm -C
    ifconfig $dev down
    ;;
*)
    echo "Usage:$(basename $0) start|stop"
    exit 1
    ;;
esac

[root@VS ~]# bash lvs_dr_vs.sh start

ldirectord实现LVS的高可用性

当RS服务端崩溃时，自动从LVS中删除

VS

[root@VS ~]# iptables -t mangle -A PREROUTING -d 192.168.80.100 -p tcp -m multiport --dports 80,443 -j MARK --set-mark 10

下载ldirectord软件包(pub/Source/7.x86/crmsh/)
[root@VS ~]# yum -y install ldirectord-3.9.6-0rc1.1.1.x86_64.rpm       \\需有完整yum源
[root@VS ~]# rpm -ql ldirectord
/etc/ha.d
/etc/ha.d/resource.d
/etc/ha.d/resource.d/ldirectord
/etc/logrotate.d/ldirectord
/usr/lib/ocf/resource.d/heartbeat/ldirectord
/usr/lib/systemd/system/ldirectord.service
/usr/sbin/ldirectord
/usr/share/doc/ldirectord-3.9.6
/usr/share/doc/ldirectord-3.9.6/COPYING
/usr/share/doc/ldirectord-3.9.6/ldirectord.cf
/usr/share/man/man8/ldirectord.8.gz
[root@VS ~]# cp /usr/share/doc/ldirectord-3.9.6/ldirectord.cf /etc/ha.d
[root@VS ~]# vim /etc/ha.d/ldirectord.cf
    checktimeout=3 \\超时时间
    checkinterval=1 \\检查间隔
    fallback=127.0.0.1:80 \\Sorry Server,错误的网页
    autoreload=yes \\自动加载配置文件
    logfile="/var/log/ldirectord.log" \\日志文件
    quiescent=no \\当RS宕机时是否将RS记录从ipvsadm记录中删除，no表示宕机即删除
    virtual=192.168.80.100:80 \\VS服务端IP
    real=192.168.198.138:80 gate 2 \\RS服务端IP，gate表示dr类型
    real=192.168.198.132:80 gate 1 \\RS服务端IP，gate表示dr类型
    fallback=127.0.0.1:80 gate
    service=http
    scheduler=wrr \\调度算法
    protocol=tcp \\tcp协议
    checktype=negotiate
    checkport=80 \\检查端口
    request="index.html" \\检查网页
    receive="danran" \\检查网页字符，若包含该字符，则表示RS服务端正常
[root@VS ~]# systemctl start ldirectord
[root@VS ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
  TCP  192.168.80.100:80 rr
  -> 192.168.198.132:80           Route   1      0          0
  -> 192.168.198.138:80           Route   1      0          0

client

[root@client ~]# curl 192.168.80.100
welcome to RS2
[root@client ~]# curl 192.168.80.100
welcome to RS1
[root@client ~]# curl 192.168.80.100
welcome to RS2
[root@client ~]# curl 192.168.80.100
welcome to RS1

使用标签实现ldirectord将多个服务定义为一个集群服务

使用打标签时需删除protocol=tcp选项

[root@VS ~]# iptables -t mangle -A PREROUTING -d 192.168.80.100 -p tcp -m multiport --dports 80,443 -j MARK --set-mark 10 \\标签定义为10
[root@VS ~]# iptables -t mangle -nvL
Chain PREROUTING (policy ACCEPT 41 packets, 3944 bytes)
pkts bytes target     prot opt in     out     source               destination
0     0 MARK       tcp  --  *      *       0.0.0.0/0            192.168.80.100       multiport dports 80,443 MARK set 0xa

[root@VS ~]# vim /etc/ha.d/ldirectord.cf
    checktimeout=3 \\超时时间
    checkinterval=1 \\检查间隔
    fallback=127.0.0.1:80 \\Sorry Server,错误的网页
    autoreload=yes \\自动加载配置文件
    logfile="/var/log/ldirectord.log" \\日志文件
    quiescent=no \\当RS宕机时是否将RS记录从ipvsadm记录中删除，no表示宕机即删除
    virtual=10  \\VS标签为10
        real=192.168.198.138:80 gate 2 \\RS服务端IP，gate表示dr类型
        real=192.168.198.132:80 gate 1 \\RS服务端IP，gate表示dr类型
        fallback=127.0.0.1:80 gate
        service=http
        scheduler=wrr \\调度算法
        checktype=negotiate
        checkport=80 \\检查端口
        request="index.html" \\检查网页
        receive="danran" \\检查网页字符，若包含该字符，则表示RS服务端正常
[root@VS ~]# systemctl start ldirectord
[root@VS ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
  FWM  10 rr
  -> 192.168.198.132:80           Route   1      0          0
  -> 192.168.198.138:80           Route   1      0          0