高可用负载均衡 haproxy+keepalived

服务器
20.0.0.206    10.0.0.206 bs-hk-hk01 高可用负载均衡节点 2c2g
20.0.0.207    10.0.0.207 bs-hk-hk02 高可用负载均衡节点 2c2g
软件版本
Keepalived 2.0.20
haproxy 2.1.2
Keepalived 安装配置
两个节点都安装   以bs-hk-hk01为例
#安装依赖包
[root@bs-hk-hk01 tools]#yum -y install gcc openssl-devel libnl3-devel pcre-devel
[root@bs-hk-hk01 tools]# ls
haproxy-2.1.2.tar.gz  keepalived-2.0.20.tar.gz
[root@bs-hk-hk01 tools]# tar -zvxf keepalived-2.0.20.tar.gz
[root@bs-hk-hk01 keepalived-2.0.20]# ./configure --prefix=/usr/local/keepalived-2.0.20
[root@bs-hk-hk01 keepalived-2.0.20]# echo $?
0
[root@bs-hk-hk01 keepalived-2.0.20]# make && make install
[root@bs-hk-hk01 keepalived-2.0.20]# echo $?
0
#配置文件放在默认路径
[root@bs-hk-hk01 keepalived-2.0.20]# mkdir /etc/keepalived/
[root@bs-hk-hk01 keepalived-2.0.20]# cp /usr/local/keepalived-2.0.20/etc/keepalived/keepalived.conf /etc/keepalived/
#keepalived启动脚本变量引用文件，默认文件路径是/etc/sysconfig/，也可以不做软链接，直接修改启动脚本中文件路径即可（安装目录下）
[root@bs-hk-hk01 keepalived-2.0.20]# cp /usr/local/keepalived-2.0.20/etc/sysconfig/keepalived /etc/sysconfig/
[root@bs-hk-hk01 keepalived-2.0.20]# cp /tools/keepalived-2.0.20/keepalived/keepalived.service /etc/systemd/system/
# 将keepalived主程序加入到环境变量（安装目录下）
[root@bs-hk-hk01 keepalived-2.0.20]# ln -s /usr/local/keepalived-2.0.20/sbin/keepalived /usr/sbin/
#keepalived启动脚本，这个从keepalived源码目录复制，安装目录中没有
[root@bs-hk-hk01 keepalived-2.0.20]# cp /tools/keepalived-2.0.20/keepalived/etc/init.d/keepalived /etc/init.d/
[root@bs-hk-hk01 keepalived-2.0.20]# chmod 755 /etc/init.d/keepalived
[root@bs-hk-hk01 keepalived-2.0.20]# systemctl enable keepalived.service
Created symlink from /etc/systemd/system/multi-user.target.wants/keepalived.service to /etc/systemd/system/keepalived.service.
[root@bs-hk-hk01 keepalived-2.0.20]# mkdir /var/log/keepalived
[root@bs-hk-hk01 keepalived-2.0.20]# vim /etc/sysconfig/keepalived
KEEPALIVED_OPTIONS="-D -d -S 0"
[root@bs-hk-hk01 keepalived-2.0.20]# vim /etc/rsyslog.d/keepalived.conf
local0.* /var/log/keepalived/keepalived.log
#修改 /etc/rsyslog.conf 文件，添加如下：
[root@bs-hk-hk01 log]# diff /etc/rsyslog.conf{,.bak}
74c74
< local0.*     /var/log/keepalived/keepalived.log
---
> 

[root@bs-hk-hk01 keepalived-2.0.20]# systemctl restart rsyslog
[root@bs-hk-hk01 keepalived-2.0.20]# systemctl start keepalived.service
[root@bs-hk-hk01 keepalived]# cp /lib/systemd/system/keepalived.service{,.bak}
[root@bs-hk-hk01 keepalived]# !vim
vim /lib/systemd/system/keepalived.service
[root@bs-hk-hk01 keepalived]# diff /lib/systemd/system/keepalived.service{,.bak}
10,11c10,11
< EnvironmentFile=/etc/sysconfig/keepalived
< ExecStart=/sbin/keepalived $KEEPALIVED_OPTIONS
---
> EnvironmentFile=-/usr/local/keepalived-2.0.20/etc/sysconfig/keepalived
> ExecStart=/usr/local/keepalived-2.0.20/sbin/keepalived $KEEPALIVED_OPTIONS
[root@bs-hk-hk01 keepalived]# systemctl daemon-reload
[root@bs-hk-hk01 keepalived]# systemctl restart keepalived.service 

keepalived 配置
#bs-hk-hk01
[root@bs-hk-hk01 keepalived]# cat keepalived.conf
! Configuration File for keepalived
! By zisefeizhu

vrrp_script chk_haproxy_port {
    script "/service/scripts/chk_hapro.sh"
    interval 2
    weight -5
    fall 2
    rise 1
}

vrrp_instance kubernetes_master {
    state MASTER
    interface eth0
    virtual_router_id 1
    priority 150
    advert_int 1
    unicast_src_ip 20.0.0.206
    unicast_peer {
        20.0.0.207
    }
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        20.0.0.250 dev eth0 label eth0:1
    }
    track_script {
        chk_haproxy_port
    }
}

#bs-hk-hk02
[root@bs-hk-hk02 keepalived]# cp keepalived.conf{,.bak}
[root@bs-hk-hk01 keepalived]# scp keepalived.conf 20.0.0.207:/etc/keepalived/keepalived.conf
[root@bs-hk-hk02 keepalived]# cat keepalived.conf
! Configuration File for keepalived
! By zisefeizhu

vrrp_script chk_haproxy_port {
    script "/service/scripts/chk_hapro.sh"
    interval 2
    weight -5
    fall 2
    rise 1
}

vrrp_instance kubernetes_master {
    state BACKUP
    interface eth0
    virtual_router_id 1
    priority 70
    advert_int 1
    unicast_src_ip 20.0.0.207
    unicast_peer {
        20.0.0.206
    }
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        20.0.0.250 dev eth0 label eth0:1
    }
    track_script {
        chk_haproxy_port
    }
}
[root@bs-hk-hk02 keepalived]# systemctl restart keepalived.service 

测试
[root@bs-hk-hk01 keepalived]# systemctl stop keepalived.service
[root@bs-hk-hk02 keepalived]# hostname -I
20.0.0.207 20.0.0.250 10.0.0.207
[root@bs-hk-hk01 keepalived]# systemctl restart keepalived.service
[root@bs-hk-hk01 keepalived]# hostname -I
20.0.0.206 20.0.0.250 10.0.0.206
[root@bs-hk-hk02 keepalived]# hostname -I
20.0.0.207 10.0.0.207
# systemctl enable keepalived.service

Haproxy 安装配置
两个节点都安装   以bs-hk-hk01为例
[root@bs-hk-hk01 ~]# yum install  vim iotop bc gcc gcc-c++ glibc glibc-devel pcre pcre-devel openssl  openssl-devel zip unzip zlib-devel  net-tools lrzsz tree ntpdate telnet lsof tcpdump wget libevent libevent-devel bc  systemd-devel bash-completion traceroute libtermcap-devel ncurses-devel libevent-devel readline-devel  -y
#LUA编译安装及Systemd开发包
[root@bs-hk-hk01 ~]# cd /tools/
[root@bs-hk-hk01 tools]# curl -R -O http://www.lua.org/ftp/lua-5.3.5.tar.gz
[root@bs-hk-hk01 tools]# ls
haproxy-2.1.2.tar.gz  keepalived-2.0.20  keepalived-2.0.20.tar.gz  lua-5.3.5.tar.gz
[root@bs-hk-hk01 tools]# tar -zxvf lua-5.3.5.tar.gz
[root@bs-hk-hk01 lua-5.3.5]# make linux
[root@bs-hk-hk01 lua-5.3.5]# echo $?
0
[root@bs-hk-hk01 lua-5.3.5]# make INSTALL_TOP=/usr/local/lua install
[root@bs-hk-hk01 lua-5.3.5]# echo $?
0
[root@bs-hk-hk01 lua-5.3.5]# yum install systemd-devel

#安装haproxy
[root@bs-hk-hk01 lua-5.3.5]# cd ..
[root@bs-hk-hk01 tools]# ls
haproxy-2.1.2.tar.gz  keepalived-2.0.20  keepalived-2.0.20.tar.gz  lua-5.3.5  lua-5.3.5.tar.gz
[root@bs-hk-hk01 tools]# tar xf haproxy-2.1.2.tar.gz
[root@bs-hk-hk01 tools]# cd haproxy-2.1.2/
[root@bs-hk-hk01 haproxy-2.1.2]# make -j $(nproc) TARGET=linux-glibc USE_OPENSSL=1 USE_ZLIB=1 USE_LUA=1 LUA_LIB=/usr/local/lua/lib/ LUA_INC=/usr/local/lua/include/ USE_PCRE=1 USE_SYSTEMD=1 PREFIX=/usr/local/haproxy
[root@bs-hk-hk01 haproxy-2.1.2]# make install PREFIX=/usr/local/haproxy
[root@bs-hk-hk01 haproxy-2.1.2]# cp haproxy /usr/sbin/
#准备启动脚本
[root@bs-hk-hk01 haproxy-2.1.2]# vim /usr/lib/systemd/system/haproxy.service
[Unit]
Description=HAProxy Load Balancer
After=syslog.target network.target
[Service]
ExecStartPre=/usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -c -q
ExecStart=/usr/sbin/haproxy -Ws -f /etc/haproxy/haproxy.cfg -p /var/lib/haproxy/haproxy.pid
ExecReload=/bin/kill -USR2 $MAINPID
[Install]
WantedBy=multi-user.target
#配置文件
[root@bs-hk-hk01 haproxy-2.1.2]# cd /etc/haproxy/
[root@bs-hk-hk01 log]# cat /etc/haproxy/haproxy.cfg
#---------------------------------------------------------------------
# Global settings
# By   zisefeizhu
#---------------------------------------------------------------------
global
maxconn 10000
uid 99
gid 99
daemon
nbproc 2
cpu-map 1 0
cpu-map 2 1
log 127.0.0.1 local3 info
#log 127.0.0.1 local1 notice

#默认配置
defaults
    log global
    mode http
    #option http-keep-alive
    option httpclose
    option  forwardfor
    option abortonclose
    maxconn 10000
    mode http
    timeout connect 300000ms
    timeout client  300000ms
    timeout server  300000ms
  balance leastconn  

#统计页面配置
listen stats
  bind 0.0.0.0:9999
    mode http
  option httplog
  maxconn 10
  stats refresh 30s
  stats enable
  log global
  stats uri     /haproxy-status
  stats realm zisefeizhu\ Haproxy
  stats auth    admin:zisefeizhu
  stats hide-version

#K8S-API-Server
frontend K8S_API
    bind *:8443
    mode tcp
    default_backend k8s_api_nodes_6443

backend k8s_api_nodes_6443
    mode tcp
    balance leastconn
    server 20.0.0.200  20.0.0.200:6443  check inter 2000 fall 3 rise 5
    server 20.0.0.201  20.0.0.201:6443  check inter 2000 fall 3 rise 5
    server 20.0.0.202  20.0.0.202:6443  check inter 2000 fall 3 rise 5

[root@bs-hk-hk01 haproxy]# mkdir /var/lib/haproxy
[root@bs-hk-hk01 haproxy]# chown 99.99 /var/lib/haproxy/ -R
[root@bs-hk-hk01 haproxy]# vim /etc/rsyslog.conf
# 对如下两行取消注释
$ModLoad imudp
$UDPServerRun 514
# 在末尾添加如下行
local3.* /var/log/haproxy/haproxy.log
*.info;mail.none;authpriv.none;cron.none;local0.none;local3.none
[root@bs-hk-hk01 log]# systemctl restart rsyslog
[root@bs-hk-hk01 log]# systemctl enable haproxy.service
Created symlink from /etc/systemd/system/multi-user.target.wants/haproxy.service to /usr/lib/systemd/system/haproxy.service.
[root@bs-hk-hk01 haproxy]# systemctl start haproxy.service 

设置服务启动顺序及依赖关系
[root@bs-hk-hk01 keepalived]# diff /usr/lib/systemd/system/keepalived.service{,.bak02}
3c3
< After=network-online.target syslog.target haproxy.service
---
> After=network-online.target syslog.target
5d4
< Requires=haproxy.service
[root@bs-hk-hk01 keepalived]# scp /usr/lib/systemd/system/keepalived.service 20.0.0.207:/usr/lib/systemd/system/keepalived.service

检查脚本
[root@bs-hk-hk01 keepalived]# mkdir /service/scripts -p
[root@bs-hk-hk01 keepalived]# vim /service/scripts/chk_hapro.sh
##########################################################################
#Author:                     zisefeizhu
#QQ:                         2********0
#Date:                       2020-02-02
#FileName:                   /service/scripts/chk_hapro.sh
#URL:                        https://www.cnblogs.com/zisefeizhu/
#Description:                The test script
#Copyright (C):              2020 All rights reserved
##########################################################################
#!/bin/bash
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin
export $PATH
counts=$(ps -ef|grep -w "haproxy"|grep -v grep|wc -l)
if [ "${counts}" = "0" ]; then
    systemctl restart keepalived.service
    sleep 2
    counts=$(ps -ef|grep -w "haproxy"|grep -v grep|wc -l)
    if [ "${counts}" = "0" ]; then
        systemctl stop keepalived.service
    fi
fi

查看日志
[root@bs-hk-hk01 log]# tree keepalived/  haproxy/
keepalived/
└── keepalived.log
haproxy/
└── haproxy.log

0 directories, 2 files

完成！