nginx+keepalived双主高可用负载均衡

实验环境及软件版本：
CentOS版本： 6.6(2.6.32.-504.el6.x86_64)
nginx版本： nginx-1.6.3
keepalived版本：keepalived-1.2.7

主LB1：LB-110-05

主LB2：LB-111-06

一、安装准备及依赖（用SecureCRT的交互窗口同时对两台LB操作，只贴出LB1的操作过程在此）

[root@LB-110-05 ~]# mkdir tools

[root@LB-110-05 ~]# mkdir /application

[root@LB-110-05 ~]# yum -y install pcre pcre-devel openssl openssl-devel gcc gcc-c++ make automake popt-devel

[root@LB-110-05 ~]# cd tools

[root@LB-110-05 tools]# tar xf nginx-1.6.3.tar.gz

二、Nginx+keepalived安装实战（用SecureCRT的交互窗口同时对两台LB操作，只贴出LB1的操作过程在此）

1. 安装nginx

[root@LB-110-05 tools]# cd nginx-1.6.3

[root@LB-110-05 nginx-1.6.3]# useradd nginx -s /sbin/nologin -M

[root@LB-110-05 nginx-1.6.3]# ./configure --prefix=/application/nginx-1.6.3 --with-http_stub_status_module --with-http_ssl_module --user=nginx --group=nginx

[root@LB-110-05 nginx-1.6.3]# echo $?   #检查上一步操作是否正确，正确返回结果为0，反之为1
0

[root@LB-110-05 nginx-1.6.3]# make && make install

[root@LB-110-05 nginx-1.6.3]# ln -s /application/nginx-1.6.3 /application/nginx

[root@LB-110-05 nginx-1.6.3]# /application/nginx/sbin/nginx -t  #检查nginx的语法是否正确和测试是否成功
nginx: the configuration file /application/nginx-1.6.3/conf/nginx.conf syntax is ok
nginx: configuration file /application/nginx-1.6.3/conf/nginx.conf test is successful

[root@LB-110-05 nginx-1.6.3]# /application/nginx/sbin/nginx   #启动nginx服务

[root@LB-110-05 nginx-1.6.3]# netstat -tunlp|grep 80   #检查nginx服务是否启动成功

tcp              0              0 0.0.0.0:80              0.0.0.0:*             LISTEN            4329/nginx

2. 安装keepalived

[root@LB-110-05 nginx-1.6.3]# cd..

[root@LB-110-05 tools]# tar xf keepalived-1.2.7.tar.gz

[root@LB-110-05 tools]# cd keepalived-1.2.7

[root@LB-110-05 keepalived-1.2.7]# ./configure

[root@LB-110-05 keepalived-1.2.7]# make && make install

[root@LB-110-05 keepalived-1.2.7]# cp /usr/local/etc/rc.d/init.d/keepalived /etc/rc.d/init.d/

[root@LB-110-05 keepalived-1.2.7]# cp /usr/local/etc/sysconfig/keepalived /etc/sysconfig/

[root@LB-110-05 keepalived-1.2.7]# mkdir /etc/keepalived

[root@LB-110-05 keepalived-1.2.7]# cp /usr/local/etc/keepalived/keepalived.conf /etc/keepalived/

[root@LB-110-05 keepalived-1.2.7]# cp /usr/local/sbin/keepalived /usr/sbin/

3. 加入开机启动

[root@LB-110-05 ~]# cat >>/etc/rc.local<<EOF
> /usr/local/nginx/sbin/nginx
> /etc/init.d/keepalived start
> EOF

[root@LB-110-05 ~]# cat /etc/rc.local
#!/bin/sh
#
# This script will be executed *after* all the other init scripts.
# You can put your own initialization stuff in here if you don't
# want to do the full Sys V style init stuff.

touch /var/lock/subsys/local
/application/nginx/sbin/nginx
/etc/init.d/keepalived start

三、配置nginx+keepalived

1. 配置nginx

[root@LB-110-05 ~]# cd /application/nginx/conf

[root@LB-110-05 conf]# cp nginx.conf nginx.conf.bak

[root@LB-110-05 conf]# vi nginx.conf

user nginx nginx;
worker_processes 1;
events {
           worker_connections 1024;
     }

http {
        include mime.types;
        default_type application/octet-stream;
        sendfile on;
        keepalive_timeout 65;

upstream myserver{
                      ip_hash;       #用ip哈希算法保持会话
                      server 10.0.0.7:80 max_fails=3 fail_timeout=20s;
                      server 10.0.0.8:80 max_fails=3 fail_timeout=20s;
            }
server {
           listen 80;
           server_name 192.168.0.110;
           location / {
           index index.php index.htm index.html;
           proxy_redirect off;
           proxy_set_header Host $host;
           proxy_set_header X-Real-IP $remote_addr;
           proxy_next_upstream http_500 http_502 http_503 error timeout invalid_header;
           proxy_set_header X-Forwarded-For $remote_addr;
           proxy_pass http://myserver;
             }
      }
}

保存退出，把LB1的nginx.conf配置文件用scp推送到LB2的/application/nginx/conf目录下就行。

[root@LB-110-05 conf]# /application/nginx/sbin/nginx -t  
nginx: the configuration file /application/nginx-1.6.3/conf/nginx.conf syntax is ok
nginx: configuration file /application/nginx-1.6.3/conf/nginx.conf test is successful

[root@LB-110-05 conf]# /application/nginx/sbin/nginx -s reload  #平滑重启nginx,不影响服务使用，提高用户体验

[root@LB-110-05 conf]# ps -ef|grep nginx|grep -v grep   #检查nginx服务是否启动成功
root 4329 1 0 17:08 ? 00:00:00 nginx: master process /application/nginx/sbin/nginx
nginx 6330 4329 0 18:01 ? 00:00:00 nginx: worker process

2. 配置keepalived

2.1 LB1的keepalived配置

[root@LB-110-05 ~]# cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak

[root@LB-110-05 ~]# vi /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
     notification_email {
         acassen@firewall.loc
         failover@firewall.loc
         sysadmin@firewall.loc
         }
     notification_email_from Alexandre.Cassen@firewall.loc
     smtp_server 192.168.200.1
     smtp_connect_timeout 30
     router_id Nginx_DEVEL
}

vrrp_instance Nginx_HA1 {
      state MASTER
      interface eth1
      virtual_router_id 51
      priority 100
      advert_int 1
      authentication {
          auth_type PASS
          auth_pass 1111
      }
      virtual_ipaddress {
             192.168.0.110/24 dev eth1
       }
}
vrrp_instance Nginx_HA2 {
      state BACKUP
      interface eth1
      virtual_router_id 52
      priority 99
      advert_int 1
      authentication {
            auth_type PASS
            auth_pass 1111
      }
      virtual_ipaddress {
         192.168.0.111/24 dev eth1
      }
}

2.2 LB2的keepalived配置

[root@LB-111-06 ~]# vi /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
        notification_email {
        acassen@firewall.loc
        failover@firewall.loc
        sysadmin@firewall.loc
        }
        notification_email_from Alexandre.Cassen@firewall.loc
        smtp_server 192.168.200.1
        smtp_connect_timeout 30
        router_id Nginx_DEVEL
}

vrrp_instance Nginx_HA1 {
      state BACKUP
      interface eth2
      virtual_router_id 51
      priority 99
      advert_int 1
      authentication {
             auth_type PASS
             auth_pass 1111
       }
       virtual_ipaddress {
               192.168.0.110/24 dev eth2
       }
}
vrrp_instance Nginx_HA2 {
      state MASTER
      interface eth2
      virtual_router_id 52
      priority 100
      advert_int 1
      authentication {
              auth_type PASS
              auth_pass 1111
      }
      virtual_ipaddress {
               192.168.0.111/24 dev eth2
      }
}

3. 启动keepalived

[root@LB-110-05~]# /etc/init.d/keepalived start   #先启动主LB1

[root@LB-111-06 ~]# /etc/init.d/keepalived start  #随后再启动LB2

四、查看服务启动成功后的结果

[root@LB-110-05 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:5c:2d:57 brd ff:ff:ff:ff:ff:ff
inet 192.168.0.110/24 brd 192.168.0.255 scope global eth0
inet6 fe80::20c:29ff:fe5c:2d57/64 scope link
valid_lft forever preferred_lft forever
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:5c:2d:61 brd ff:ff:ff:ff:ff:ff
inet 10.0.0.5/24 brd 10.0.0.255 scope global eth1
inet 192.168.0.110/24 scope global eth1
inet6 fe80::20c:29ff:fe5c:2d61/64 scope link
valid_lft forever preferred_lft forever

[root@LB-111-06 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:50:8e:3a brd ff:ff:ff:ff:ff:ff
inet 192.168.0.111/24 brd 192.168.0.255 scope global eth1
inet6 fe80::20c:29ff:fe50:8e3a/64 scope link
valid_lft forever preferred_lft forever
3: eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:50:8e:44 brd ff:ff:ff:ff:ff:ff
inet 10.0.0.6/24 brd 10.0.0.255 scope global eth2
inet 192.168.0.111/24 scope global eth2
inet6 fe80::20c:29ff:fe50:8e44/64 scope link
valid_lft forever preferred_lft forever

五、测试高可用

[root@LB-111-06 ~]# /etc/init.d/keepalived stop
Stopping keepalived: [ OK ]
[root@LB-111-06 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:50:8e:3a brd ff:ff:ff:ff:ff:ff
inet 192.168.0.111/24 brd 192.168.0.255 scope global eth1
inet6 fe80::20c:29ff:fe50:8e3a/64 scope link
valid_lft forever preferred_lft forever
3: eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:50:8e:44 brd ff:ff:ff:ff:ff:ff
inet 10.0.0.6/24 brd 10.0.0.255 scope global eth2
inet6 fe80::20c:29ff:fe50:8e44/64 scope link
valid_lft forever preferred_lft forever

[root@LB-110-05 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:5c:2d:57 brd ff:ff:ff:ff:ff:ff
inet 192.168.0.110/24 brd 192.168.0.255 scope global eth0
inet6 fe80::20c:29ff:fe5c:2d57/64 scope link
valid_lft forever preferred_lft forever
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:5c:2d:61 brd ff:ff:ff:ff:ff:ff
inet 10.0.0.5/24 brd 10.0.0.255 scope global eth1
inet 192.168.0.110/24 scope global eth1
inet 192.168.0.111/24 scope global secondary eth1
inet6 fe80::20c:29ff:fe5c:2d61/64 scope link
valid_lft forever preferred_lft forever