activitmq+keepalived+nfs 非zk的高可用集群构建

nfs	192.168.10.32
maast	192.168.10.4
savel	192.168.10.31

应对这个需求既要高可用又要消息延迟,只能使用变态方式实现

nfs部署

#yum install nfs-utils rpcbind
#vim /etc/exports
/data/activemq  192.168.10.31(rw,sync,no_root_squash,no_all_squash)
/data/activemq  192.168.10.4(rw,sync,no_root_squash,no_all_squash)
#systemctl stop firewalld.service
#setenforce 0
#mkdir /data/activemq -p
#systemctl start nfs

客户端挂载

#yum -y install nfs-utils.x86_64
# systemctl start rpcbind
# showmount -e 192.168.10.32
Export list for 192.168.10.32:
/data/activemq 192.168.10.4,192.168.10.31
# mkdir /data
# mount -t nfs 192.168.10.32:/data/activemq /data
# df -h
文件系统                      容量  已用  可用 已用% 挂载点
/dev/mapper/centos-root        17G  4.8G   13G   28% /
devtmpfs                      476M     0  476M    0% /dev
tmpfs                         488M     0  488M    0% /dev/shm
tmpfs                         488M  1.6M  486M    1% /run
tmpfs                         488M     0  488M    0% /sys/fs/cgroup
/dev/sda1                    1014M  130M  885M   13% /boot
tmpfs                          98M     0   98M    0% /run/user/0
192.168.10.32:/data/activemq   17G  2.0G   16G   12% /data

使用yum安装keepalived

# yum -y install epel-release-7-11.noarch
# yum -y install keepalived

安装activemq部署

# tar xf apache-activemq-5.15.8-bin.tar.gz -C /usr/local/
# cd /usr/local/apache-activemq-5.15.8/conf
# vim activemq.xml     改成这样的两个节点都一样
 40     <broker xmlns="http://activemq.apache.org/schema/core" brokerName="localhost" dataDirectory="/data" schedulerSupport="true">
 81         <persistenceAdapter>
 82             <kahaDB directory="/data/kahadb"/>
 83         </persistenceAdapter>
# ./activemq start  启动
# ss -lntp | grep 61616
LISTEN     0      128         :::61616                   :::*                   users:(("java",pid=21705,fd=137))
查另一个节点的端口
# ss -lntp | grep 61616

注意

1、共享文件的主从配置,是通过文件共享锁来实现的。先拿到文件锁的服务就是master,其它为slave服务,slave服务默认每10秒试图拿一次文件锁,可以查看activeMq的控制台。

2、只有master的控制台(如http://192.168.10.4:8161/admin)能够访问,slave的控制台不能访问,原因就是slave没有拿到文件锁,不能访问文件。

配置keepalived的文件

备节点的配置

[root@localhost conf]# cd /etc/keepalived/
[root@localhost keepalived]# cat keepalived.conf
global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from ka2@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   vrrp_mcast_group4 224.111.111.111
}

vrrp_script chk_mq {
    script "ss -lntp | grep 61616"  #检查端口是否存在，这里不能查进程
    interval 2
    weight -10
    fall 2
    rise 2
}

vrrp_instance External_1 {
    state BACKUP
    interface ens33
    virtual_router_id 171
    priority 95
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1402b1b5
    }
    virtual_ipaddress {
        192.168.10.5/24
    }
    track_script {
        chk_mq
    }
}

　　

主节点的配置

# cat /etc/keepalived/keepalived.conf
global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from ka1@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   vrrp_mcast_group4 224.111.111.111
}

vrrp_script chk_mq {
    script "ss -lntp | grep 61616"
    interval 2
    weight -10
    fall 2
    rise 2
}

vrrp_instance External_1 {
    state MASTER
    interface ens33
    virtual_router_id 171
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1402b1b5
    }
    virtual_ipaddress {
        192.168.10.5/24
    }
    track_script {
        chk_mq
    }
}

验证

[root@jiagoushi bin]# ss -lntp | grep 61616
LISTEN     0      128         :::61616                   :::*                   users:(("java",pid=21705,fd=137))
[root@jiagoushi bin]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:6a:6e:da brd ff:ff:ff:ff:ff:ff
    inet 192.168.10.4/24 brd 192.168.10.255 scope global noprefixroute ens33
       valid_lft forever preferred_lft forever
    inet 192.168.10.5/24 scope global secondary ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::13ba:653e:ffc4:7305/64 scope link noprefixroute
       valid_lft forever preferred_lft forever

重启activemq查看vip切换效果，注意谁先起谁就是主节点，主节点挂了释放文件锁后，从节点端口才能启动

[root@jiagoushi bin]# ./activemq restart
INFO: Loading '/usr/local/apache-activemq-5.15.8//bin/env'
INFO: Using java '/usr/bin/java'
INFO: Loading '/usr/local/apache-activemq-5.15.8//bin/env'
INFO: Using java '/usr/bin/java'
INFO: Waiting at least 30 seconds for regular process termination of pid '21705' :
Java Runtime: Oracle Corporation 1.8.0_181 /usr/lib/jvm/java-1.8.0-openjdk-1.8.0.181-3.b13.el7_5.x86_64/jre
  Heap sizes: current=62976k  free=61991k  max=932352k
    JVM args: -Xms64M -Xmx1G -Djava.util.logging.config.file=logging.properties -Djava.security.auth.login.config=/usr/local/apache-activemq-5.15.8//conf/login.config -Dactivemq.classpath=/us
r/local/apache-activemq-5.15.8//conf:/usr/local/apache-activemq-5.15.8//../lib/: -Dactivemq.home=/usr/local/apache-activemq-5.15.8/ -Dactivemq.base=/usr/local/apache-activemq-5.15.8/ -Dactivemq.conf=/usr/local/apache-activemq-5.15.8//conf -Dactivemq.data=/usr/local/apache-activemq-5.15.8//dataExtensions classpath:
  [/usr/local/apache-activemq-5.15.8/lib,/usr/local/apache-activemq-5.15.8/lib/camel,/usr/local/apache-activemq-5.15.8/lib/optional,/usr/local/apache-activemq-5.15.8/lib/web,/usr/local/apache
-activemq-5.15.8/lib/extra]ACTIVEMQ_HOME: /usr/local/apache-activemq-5.15.8
ACTIVEMQ_BASE: /usr/local/apache-activemq-5.15.8
ACTIVEMQ_CONF: /usr/local/apache-activemq-5.15.8/conf
ACTIVEMQ_DATA: /usr/local/apache-activemq-5.15.8/data
Connecting to pid: 21705
INFO: failed to resolve jmxUrl for pid:21705, using default JMX url
Connecting to JMX URL: service:jmx:rmi:///jndi/rmi://localhost:1099/jmxrmi
.....INFO: Broker not available at: service:jmx:rmi:///jndi/rmi://localhost:1099/jmxrmi
..........................
INFO: Regular shutdown not successful,  sending SIGKILL to process
INFO: sending SIGKILL to pid '21705'

INFO: Loading '/usr/local/apache-activemq-5.15.8//bin/env'
INFO: Using java '/usr/bin/java'
INFO: Starting - inspect logfiles specified in logging.properties and log4j.properties to get details
INFO: pidfile created : '/usr/local/apache-activemq-5.15.8//data/activemq.pid' (pid '40627')
INFO: Loading '/usr/local/apache-activemq-5.15.8//bin/env'
INFO: Using java '/usr/bin/java'
ActiveMQ is running (pid '40627')
[root@jiagoushi bin]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:6a:6e:da brd ff:ff:ff:ff:ff:ff
    inet 192.168.10.4/24 brd 192.168.10.255 scope global noprefixroute ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::13ba:653e:ffc4:7305/64 scope link noprefixroute
       valid_lft forever preferred_lft forever
[root@jiagoushi conf]#  ss -lntp | grep 61616
另一节点查看
[root@localhost keepalived]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:60:f4:68 brd ff:ff:ff:ff:ff:ff
    inet 192.168.10.31/24 brd 192.168.10.255 scope global noprefixroute ens33
       valid_lft forever preferred_lft forever
    inet 192.168.10.5/24 scope global secondary ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::2dbd:f06e:6abd:907/64 scope link noprefixroute
       valid_lft forever preferred_lft forever
[root@localhost keepalived]# ss -lntp | grep 61616
LISTEN     0      128         :::61616                   :::*                   users:(("java",pid=22702,fd=137))

　　参考文章：https://cn.aliyun.com/jiaocheng/312402.html