【Linux】【Services】【SaaS】Docker+kubernetes（10. 利用反向代理实现服务高可用）

1. 简介

1.1. 由于K8S并没有自己的集群，所以需要借助其他软件来实现，公司的生产环境使用的是Nginx，想要支持TCP转发要额外安装模块，测试环境中我就使用HAPROXY了

1.2. 由于是做实现，我用Nginx又重新实现了一下HAPROXY的TCP转发，都会在本文中介绍

1.3. Haproxy配置参数详解请参考我前面的文章：http://www.cnblogs.com/demonzk/p/6904029.html

2. 环境

2.1. 机器列表

功能与组件	机器名	服务IP	管理IP	VIP	Processor Cores	RAM	Storage	备注
HAPROXY	HCTJOSDR01	10.30.2.48	172.16.0.48	172.16.0.148
HAPROXY	HCTJOSDR02	10.30.2.49	172.16.0.49	172.16.0.149

2.2. 架构图

2.3. 软件版本

haproxy　　　　1.5.18-6.el7

keepalived　　　　1.3.5-1.el7

nginx　　　　　　1.12.2-1.el7

3. 安装与基础配置

3.1. haproxy

yum安装

yum -y install haproxy

配置haproxy日志，修改/etc/rsyslog.conf

#去掉下面两行的注释
$ModLoad imudp
$UDPServerRun 514

添加一个配置文件/etc/rsyslog.d/haproxy.conf

local2.*                       /var/log/haproxy.log

修改/etc/sysconfig/rsyslog

#-r是允许接受外部日志
#-c 是说兼容syslog v2
#-m 是说每隔多长时间加一个时间戳，0表示不加
SYSLOGD_OPTIONS="-r -c 2"

修改haproxy配置文件，删掉没用的，添加状态监控页面

global
    # to have these messages end up in /var/log/haproxy.log you will
    # need to:
    #
    # 1) configure syslog to accept network log events.  This is done
    #    by adding the '-r' option to the SYSLOGD_OPTIONS in
    #    /etc/sysconfig/syslog
    #
    # 2) configure local2 events to go to the /var/log/haproxy.log
    #   file. A line like the following can be added to
    #   /etc/sysconfig/syslog
    #
    #    local2.*                       /var/log/haproxy.log
    #
    log         127.0.0.1 local2

    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon

    # turn on stats unix socket
    stats socket /var/lib/haproxy/stats

#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000

#状态监控页面
listen stats 0.0.0.0:9001
    stats enable
    stats uri /haproxyadmin?stats
    stats realm HAProxy\ Statistics
    stats auth admin:admin
    stats admin if TRUE

启用

systemctl start haproxy && systemctl enable haproxy

3.1. 或者使用Nginx做TCP转发（本次使用Mysql做例子）

安装

yum -y install nginx

在/etc/nginx/nginx.conf中添加下面这段

stream {

    log_format tcp_proxy '$remote_addr [$time_local] '
                 '$protocol $status $bytes_sent $bytes_received '
                 '$session_time "$upstream_addr" '
                 '"$upstream_bytes_sent" "$upstream_bytes_received" "$upstream_connect_time"';
    include /etc/nginx/tcp.d/*.conf;
}

在/etc/nginx/tcp.d/mysql.hccos.cn.conf中写入如下内容

server {
    listen          3306;
    proxy_connect_timeout 5s;
    proxy_timeout 30s;
        proxy_pass mysql;
    }
upstream mysql {
    server 172.16.0.25:3306 max_fails=3 fail_timeout=10s;
    server 172.16.0.26:3306 max_fails=3 fail_timeout=10s;
    server 172.16.0.27:3306 max_fails=3 fail_timeout=10s;
}

3.2. keepalived安装

yum安装

yum -y install keepalived

修改/etc/sysconfig/keepalived

KEEPALIVED_OPTIONS="-D -d -S 0"

修改/etc/rsyslog.d/keepalived.conf

local0.*    /var/log/keepalived.log

在两台机器上修改/etc/keepalived/keepalived.conf

10.30.2.48

global_defs {
   notification_email {
     eric.zhangtj@homecredit.cn
   }
   notification_email_from eric.zhangtj@homecredit.cn
   smtp_server 10.25.8.2
   smtp_connect_timeout 30
   router_id LVS_DEVEL
}

vrrp_script check_haproxy {
script "killall -0 haproxy"
        interval 1
        weight 21
}

vrrp_script chk_mantaince_down {
   script "[[ -f /etc/keepalived/down ]] && exit 1 || exit 0"
   interval 1
   weight 2
}

vrrp_instance VI_148 {
    state MASTER
    interface ens192
    virtual_router_id 22
    garp_master_delay 1
    mcast_src_ip 172.16.0.48
    lvs_sync_daemon_interface ens192
    priority 110
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    track_interface {
    ens192
    }
    virtual_ipaddress {
        172.16.0.148/24 dev ens192 label ens192:0
    }
    track_script {
    check_haproxy
    chk_mantaince_down
    }
}

vrrp_instance VI_149 {
    state BACKUP
    interface ens192
    virtual_router_id 23
    garp_master_delay 1
    mcast_src_ip 172.16.0.49
    lvs_sync_daemon_interface ens192
    priority 100
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    track_interface {
        ens192
    }
    virtual_ipaddress {
        172.16.0.149/24 dev ens192 label ens192:1
    }
    track_script {
    check_haproxy
    chk_mantaince_down
    }
}

10.30.2.49

global_defs {
   notification_email {
     eric.zhangtj@homecredit.cn
   }
   notification_email_from eric.zhangtj@homecredit.cn
   smtp_server 10.25.8.2
   smtp_connect_timeout 30
   router_id LVS_DEVEL
}

vrrp_script check_haproxy {
script "killall -0 haproxy"
        interval 1
        weight 21
}

vrrp_script chk_mantaince_down {
   script "[[ -f /etc/keepalived/down ]] && exit 1 || exit 0"
   interval 1
   weight 2
}

vrrp_instance VI_148 {
    state BACKUP
    interface ens192
    virtual_router_id 22
    garp_master_delay 1
    mcast_src_ip 172.16.0.48
    lvs_sync_daemon_interface ens192
    priority 100
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    track_interface {
    ens192
    }
    virtual_ipaddress {
        172.16.0.148/24 dev ens192 label ens192:0
    }
    track_script {
    check_haproxy
    chk_mantaince_down
    }
}

vrrp_instance VI_49 {
    state MASTER
    interface ens192
    virtual_router_id 23
    garp_master_delay 1
    mcast_src_ip 172.16.0.49
    lvs_sync_daemon_interface ens192
    priority 110
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    track_interface {
        ens192
    }
    virtual_ipaddress {
        172.16.0.149/24 dev ens192 label ens192:1
    }
    track_script {
    check_haproxy
    chk_mantaince_down
    }
}

启用服务

systemctl start keepalived && systemctl enable keepalived

3.3. 内核参数

修改/etc/sysctl.conf

# Controls IP packet forwarding
# 开启IP转发功能
net.ipv4.ip_forward = 1

# 开启允许绑定非本机的IP
net.ipv4.ip_nonlocal_bind = 1

sysctl -p