Nginx HA 及https配置部署

 Nginx HA 

整体方案架构为:



                  (内网192.168.199.5)
             +-----------VIP----------+
             |                        |
             |                        |
           Master                   Backup
        192.168.199.90            192.168.199.57
        +----------+             +----------+
        | HAProxy  |             | HAProxy  |
        |nginx(SSL)|             |nginx(SSL)|
        |keepalived|             |keepalived|
        +----------+             +----------+
             |
             v
      192.168.199.88/89
       +----------+
       | multiple |
       |  NGINXs  |
       +----------+
             |
             v
    +--------+---------+
    |        |         |
    |        |         |
    v        v         v
+------+  +------+  +------+
| WEB1 |  | WEB2 |  | WEB3 |
+------+  +------+  +------+


各软件作用:
  * Keepalived：判定HAProxy存活，保证HA
  * HAProxy：做HTTP Load Balance
  * Nginx(SSL)：与HAProxy放置在同一服务器，负责ssl offload
  * Nginx(LB)：load balancer for app servers & web servers

客户端访问示意图:

      +--------+      HTTP                      :80 +----------+
      | client |  --------------------------------> |          |
      |        |                                    | haproxy, |
      +--------+             +---------+            |  1 or 2  |
     /        /     HTTPS    |  Nginx  |  HTTP  :80 | listening|
    <________/    ---------> |  (SSL)  | ---------> |  ports   |
                             |         |            |          |
                             +---------+            +----------+

 HAProxy + NGINX(SSL) 

使用HAProxy做HTTP的Load Balancer，使用Nginx做SSL Offload。

测试环境:
  * CentOS 6.4 x86_64 (Final)
  * Supermicro 2U4 Node
  * 域名: l99.com

IP分配:
  * lb01.l99.com 192.168.199.88
  * lb01.l99.com 192.168.199.89
  * www.l99.com 192.168.199.5 (virtual IP)
  * 192.168.199.90 做 Load Balancer (HAProxy + Nginx)

 安装配置HAProxy 


yum install libev-devel openssl-devel

cd /usr/local/src
wget http://haproxy.1wt.eu/download/1.4/src/haproxy-1.4.24.tar.gz
git clone https://github.com/cbonte/haproxy-patches.git

tar zxvf haproxy-1.4.24.tar.gz 

# 给haproxy 1.4.24 打 proxy协议补丁(haproxy 1.5之后才支持accpet-proxy, 由于我们要使用stud做ssl offload, 需要支持accept-proxy)
cd haproxy-1.4.24
patch -p1 < /usr/local/src/haproxy-patches/proxy-protocol/haproxy-1.4-proxy-protocol.patch 

make TARGETlinux2628 USE_EPOLL1 ARCHx86_64 && make install
cp /usr/local/src/haproxy-1.4.24/haproxy /usr/sbin/

cp examples/haproxy.init /etc/init.d/haproxy
chmod +x /etc/init.d/haproxy

chkconfig --add haproxy
chkconfig haproxy on

vim /etc/haproxy/haproxy.cfg


haproxy.cfg如下：

#---------------------------------------------------------------------
# Example configuration for a possible web application.  See the
# full configuration options online.
#
#   http://haproxy.1wt.eu/download/1.4/doc/configuration.txt
#
#---------------------------------------------------------------------

#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global
    # to have these messages end up in /var/log/haproxy.log you will
    # need to:
    #
    # 1) configure syslog to accept network log events.  This is done
    #    by adding the '-r' option to the SYSLOGD_OPTIONS in
    #    /etc/sysconfig/syslog
    #
    # 2) configure local2 events to go to the /var/log/haproxy.log
    #   file. A line like the following can be added to
    #   /etc/sysconfig/syslog
    #
    #    local2.*                       /var/log/haproxy.log
    #
#    log         127.0.0.1 local2
    log 127.0.0.1   local0
    log 127.0.0.1   local1 debug

    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn   45000 # Total Max Connections. This is dependent on ulimit
    user        haproxy
    group       haproxy
    daemon
    nbproc      12 # 取决于CPU处理器核数，这里的测试机是2个6核Intel E5-2620 CPU，所以核数是12

    # turn on stats unix socket
    stats socket /var/lib/haproxy/stats

#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
    mode                    http
    log                     global
    balance         roundrobin
#    balance            leastconn
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor header X-Real-IP
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         5000ms
    timeout client          50000ms
    timeout server          50000ms
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn   45000 # Total Max Connections. This is dependent on ulimit
    stats enable
    stats uri /stats # Real path redacted
    stats realm Haproxy\ Statistics
    stats auth username:password # Real credentials redacted
    monitor-uri /monitor # Returns 200 if we're up; real path redacted

frontend http-in :80
    reqdel X-Real-IP
    reqadd X-Forwarded-Proto:\ http
    default_backend http-load-balancer

frontend https-in
#    bind 127.0.0.1:8443 accept-proxy
    bind 127.0.0.1:8443
#    reqdel X-Real-IP
    reqadd X-Forwarded-Proto:\ https
    default_backend http-load-balancer

backend http-load-balancer
   server lb-1 192.168.199.88:80 maxconn 10000 check port 80
   server lb-2 192.168.199.89:80 maxconn 10000 check port 80


 安装配置Nginx(SSL) 

/usr/local/nginx/conf/nginx.conf


user  nginx;
worker_processes  12;

error_log  logs/error.log crit;

pid        logs/nginx.pid;
worker_rlimit_nofile    30000;

events {
    use epoll;
    worker_connections  51200;
}

http {
    include       mime.types;
    default_type  application/octet-stream;

    # include common options #
    include options.conf;

    # include proxy settings #
    include proxy.conf;

    # domain config #
    include l99.com/*.conf;

}


/usr/local/nginx/conf/l99.com/www.l99.com.conf

server {
        listen 443;

    ssl on;
        ssl_certificate /usr/local/nginx/conf/l99.com/lifeix-l99.crt;
        ssl_certificate_key /usr/local/nginx/conf/l99.com/lifeix-l99.key;
        ssl_client_certificate /usr/local/nginx/conf/l99.com/lifeix-dvroot.crt;
        ssl_session_timeout 5m;

        ssl_protocols SSLv2 SSLv3 TLSv1;
        ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP;
        ssl_prefer_server_ciphers on;

    default_type  text/plain;

        access_log logs/access.www.ssl.l99.com.log main;
        error_log logs/error.www.ssl.l99.com.log;
        server_name www.l99.com;

    if ($request_uri ~ update.php) {
                rewrite /(.*)$  http://www.L99.com/timeline.action last;
        }   

        location / {
                proxy_cache off;
                proxy_next_upstream http_502 http_504 error timeout invalid_header;
                proxy_ignore_headers   Expires Cache-Control;
                proxy_store         off;
                proxy_set_header        Host            $host;
                proxy_set_header        X-Real-IP       $remote_addr;
                proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
                more_clear_headers  "Cache-Control";
                add_header      Cache-Control "no-cache,max-age0";

        proxy_pass http://127.0.0.1:8443;
        }

}

 启动并测试 


service haproxy restart
service nginx restart

# 测试 HTTPS
openssl s_client -connect 192.168.199.90:443 -servername l99.com

# 测试HTTP
telnet 192.168.199.90 80
GET / HTTP/1.1
Host: www.L99.com



 Nginx(LB)配置修改 

修改options.conf (主要是由于使用HAProxy作为代理后，需要记录来源IP)

  log_format  main  '$http_x_forwarded_proto $http_x_real_ip $remote_addr $host $remote_user [$time_local] "$request" '
                    '$status $body_bytes_sent "$http_referer" "$http_user_agent" '
                    '$request_time $upstream_response_time $pipe "$gzip_ratio"';



重启nginx后，通过haproxy访问立方网日志如下:

https 192.168.199.15 192.168.199.90 www.l99.com - [04/Oct/2013:17:02:33 +0800] "GET /skin/recharge/images/paybtn_bg.jpg HTTP/1.1" 304 0 "https://www.l99.com/Recharge_pay.action" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.76 Safari/537.36" 0.007 0.006 . "-"


 HAProxy + Keepalived 

/etc/keepalived/keepalived.conf

! Configuration File for keepalived

global_defs {
   router_id LVS_DEVEL
}

vrrp_script chk_haproxy {
   script "killall -0 haproxy"   # verify the pid existance
   interval 2                    # check every 2 seconds
   weight 2                      # add 2 points of prio if OK
}

vrrp_script chk_nginx {
   script "killall -0 nginx"   # verify the pid existance
   interval 2                    # check every 2 seconds
   weight 2                      # add 2 points of prio if OK
}

vrrp_instance VI_1 {
   interface eth0                # interface to monitor
   state MASTER
   virtual_router_id 51          # Assign one ID for this route
   priority 101                  # 101 on master, 100 on backup
   virtual_ipaddress {
       192.168.199.5            # the virtual IP
   }
   track_script {
       chk_haproxy
       chk_nginx
   }
}