基于redis5的redis cluster部署

一、环境规划

#准备六台主机，地址如下
10.0.0.8   ---> master1
10.0.0.18  ---> master2
10.0.0.28  ---> master3
10.0.0.38  ---> slave1
10.0.0.48  ---> slave2
10.0.0.58  ---> slave3

二、配置redis cluster

2.1 部署redis

##安装redis
# dnf -y install redis
# redis-server --version
Redis server v=5.0.3 sha=00000000:0 malloc=jemalloc-5.1.0 bits=64 build=9529b692c0384fb7

##修改配置文件
# sed -i.bak -e 's/bind 127.0.0.1/bind 0.0.0.0/' \
-e '/# masterauth/c masterauth 123456' \
-e '/# requirepass/c requirepass 123456' \
-e '/# cluster-enabled yes/c cluster-enabled yes' \
-e '/# cluster-config-file nodes-6379.conf/c cluster-config-file nodes-6379.conf' \
-e '/# cluster-require-full-coverage/c cluster-require-full-coverage no' \
/etc/redis.conf

##验证Redis服务状态
# ss -ntl
State                    Recv-Q                   Send-Q                                       Local Address:Port                                        Peer Address:Port
LISTEN                   0                        128                                                0.0.0.0:6379                                             0.0.0.0:*
LISTEN                   0                        128                                                0.0.0.0:111                                              0.0.0.0:*
LISTEN                   0                        128                                                0.0.0.0:22                                               0.0.0.0:*
LISTEN                   0                        100                                              127.0.0.1:25                                               0.0.0.0:*
LISTEN                   0                        128                                                0.0.0.0:16379                                            0.0.0.0:*
LISTEN                   0                        128                                                   [::]:111                                                 [::]:*
LISTEN                   0                        128                                                   [::]:22                                                  [::]:*
LISTEN                   0                        100                                                  [::1]:25                                                  [::]:*

2.2 配置reis cluster

##redis-cli --cluster-replicas 1 表示每个master对应一个slave节点
# redis-cli -a 123456 --cluster create 10.0.0.8:6379 10.0.0.18:6379 10.0.0.28:6379 10.0.0.38:6379 10.0.0.48:6379 10.0.0.58:6379 --cluster-replicas 1
>>> Performing hash slots allocation on 6 nodes...
Master[0] -> Slots 0 - 5460
Master[1] -> Slots 5461 - 10922
Master[2] -> Slots 10923 - 16383
Adding replica 10.0.0.38:6379 to 10.0.0.8:6379
Adding replica 10.0.0.48:6379 to 10.0.0.18:6379
Adding replica 10.0.0.58:6379 to 10.0.0.28:6379
M: 7a4443e815ca6009d8e25e5f84d2f5feee3c007a 10.0.0.8:6379	##带M的为master
   slots:[0-5460] (5461 slots) master		##当前master的槽位起始和结束位
M: 2f41d6b0a3b8957755b67b2dccfb764dee57f123 10.0.0.18:6379
   slots:[5461-10922] (5462 slots) master
M: 4e0eb42d503974e00962fa1d0214593720327d4a 10.0.0.28:6379
   slots:[10923-16383] (5461 slots) master
S: 5308ffa51d731096d8c9e86d2f3976cb94fa7fa8 10.0.0.38:6379	##带S的slave
   replicates 7a4443e815ca6009d8e25e5f84d2f5feee3c007a
S: 67eb5b5226f01b6dde9145fe4063b1f4849fe921 10.0.0.48:6379
   replicates 2f41d6b0a3b8957755b67b2dccfb764dee57f123
S: 128518070585eb054210b322b8a6f910cc2849f8 10.0.0.58:6379
   replicates 4e0eb42d503974e00962fa1d0214593720327d4a
Can I set the above configuration? (type 'yes' to accept): yes	##输入yes自动创建集群
>>> Nodes configuration updated
>>> Assign a different config epoch to each node
>>> Sending CLUSTER MEET messages to join the cluster
Waiting for the cluster to join
......
>>> Performing Cluster Check (using node 10.0.0.8:6379)
M: 7a4443e815ca6009d8e25e5f84d2f5feee3c007a 10.0.0.8:6379
   slots:[0-5460] (5461 slots) master
   1 additional replica(s)
S: 67eb5b5226f01b6dde9145fe4063b1f4849fe921 10.0.0.48:6379
   slots: (0 slots) slave
   replicates 2f41d6b0a3b8957755b67b2dccfb764dee57f123
M: 2f41d6b0a3b8957755b67b2dccfb764dee57f123 10.0.0.18:6379
   slots:[5461-10922] (5462 slots) master
   1 additional replica(s)
S: 128518070585eb054210b322b8a6f910cc2849f8 10.0.0.58:6379
   slots: (0 slots) slave
   replicates 4e0eb42d503974e00962fa1d0214593720327d4a
S: 5308ffa51d731096d8c9e86d2f3976cb94fa7fa8 10.0.0.38:6379
   slots: (0 slots) slave
   replicates 7a4443e815ca6009d8e25e5f84d2f5feee3c007a
M: 4e0eb42d503974e00962fa1d0214593720327d4a 10.0.0.28:6379
   slots:[10923-16383] (5461 slots) master
   1 additional replica(s)
[OK] All nodes agree about slots configuration.	##所有节点槽位分配完成
>>> Check for open slots...	##检查打开的槽位
>>> Check slots coverage...	##检查插槽覆盖范围
[OK] All 16384 slots covered.	#所有槽位(16384个)分配完成

###观察以上结果，可以看到3组master/slave
master:10.0.0.8---slave:10.0.0.38
master:10.0.0.18---slave:10.0.0.48
master:10.0.0.28---slave:10.0.0.58

2.3 查看主从状态、集群状态

# redis-cli -a 123456 -h 10.0.0.18 --no-auth-warning cluster nodes

2.4 模拟故障

##模拟master1(IP：10.0.0.8)节点出故障
# redis-cli -a 123456  shutdown		##关闭master1
# tail -f /var/log/redis/redis.log
2300:C 23 Jan 2022 12:11:41.952 * DB saved on disk
2300:C 23 Jan 2022 12:11:41.952 * RDB: 4 MB of memory used by copy-on-write
2296:M 23 Jan 2022 12:11:42.027 * Background saving terminated with success
2296:M 23 Jan 2022 12:11:42.028 * Synchronization with replica 10.0.0.38:6379 succeeded
2296:M 23 Jan 2022 12:11:43.081 # Cluster state changed: ok
2296:M 23 Jan 2022 12:14:45.924 # User requested shutdown...
2296:M 23 Jan 2022 12:14:45.924 * Saving the final RDB snapshot before exiting.
2296:M 23 Jan 2022 12:14:45.927 * DB saved on disk
2296:M 23 Jan 2022 12:14:45.927 * Removing the pid file.
2296:M 23 Jan 2022 12:14:45.927 # Redis is now ready to exit, bye bye...

##查看角色变化
# redis-cli -a 123456 -h 10.0.0.38 --no-auth-warning cluster nodes
4e0eb42d503974e00962fa1d0214593720327d4a 10.0.0.28:6379@16379 master - 0 1636914427274 3 connected 10923-16383
2f41d6b0a3b8957755b67b2dccfb764dee57f123 10.0.0.18:6379@16379 master - 0 1636914429452 2 connected 5461-10922
128518070585eb054210b322b8a6f910cc2849f8 10.0.0.58:6379@16379 slave 4e0eb42d503974e00962fa1d0214593720327d4a 0 1636914423993 6 connected
67eb5b5226f01b6dde9145fe4063b1f4849fe921 10.0.0.48:6379@16379 slave 2f41d6b0a3b8957755b67b2dccfb764dee57f123 0 1636914430542 5 connected
7a4443e815ca6009d8e25e5f84d2f5feee3c007a 10.0.0.8:6379@16379 master,fail - 1636913491097 1636913490340 1 disconnected
5308ffa51d731096d8c9e86d2f3976cb94fa7fa8 10.0.0.38:6379@16379 myself,master - 0 0 8 connected 0-5460

##查看slave1的日志
# tail -f /var/log/redis/redis.log
2050:S 15 Nov 2021 02:11:47.767 * FAIL message received from 2f41d6b0a3b8957755b67b2dccfb764dee57f123 about 7a4443e815ca6009d8e25e5f84d2f5feee3c007a
2050:S 15 Nov 2021 02:11:47.867 # Start of election delayed for 795 milliseconds (rank #0, offset 238).
2050:S 15 Nov 2021 02:11:48.733 * Connecting to MASTER 10.0.0.8:6379
2050:S 15 Nov 2021 02:11:48.734 * MASTER <-> REPLICA sync started
2050:S 15 Nov 2021 02:11:48.734 # Starting a failover election for epoch 8.
2050:S 15 Nov 2021 02:11:48.736 # Error condition on socket for SYNC: Connection refused
2050:S 15 Nov 2021 02:11:48.738 # Failover election won: I'm the new master.  ##已经变成新的master
2050:S 15 Nov 2021 02:11:48.738 # configEpoch set to 8 after successful failover
2050:M 15 Nov 2021 02:11:48.738 # Setting secondary replication ID to 68e9f4f70b4fec8086c0717b8981d5adca478be0, valid up to offset: 239. New replication ID is 22f817fbe0ef49a3c0ef26e5a34ccd1f38743bde
2050:M 15 Nov 2021 02:11:48.738 * Discarding previously cached master state.

2.5 恢复故障

# systemctl start redis

# redis-cli  -a 123456 -h 10.0.38 info replication
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
# Replication
role:master
connected_slaves:1
slave0:ip=10.0.0.8,port=6379,state=online,offset=644,lag=1
master_replid:22f817fbe0ef49a3c0ef26e5a34ccd1f38743bde
master_replid2:68e9f4f70b4fec8086c0717b8981d5adca478be0
master_repl_offset:658
second_repl_offset:239
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:658

# redis-cli  -a 123456 -h 10.0.8 info replication
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
# Replication
role:slave
master_host:10.0.0.38
master_port:6379
master_link_status:up
master_last_io_seconds_ago:5
master_sync_in_progress:0
slave_repl_offset:826
slave_priority:100
slave_read_only:1
connected_slaves:0
master_replid:22f817fbe0ef49a3c0ef26e5a34ccd1f38743bde
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:826
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:239
repl_backlog_histlen:588

##查看最终master/slave关系
# redis-cli -a 123456 -h 10.0.0.8 --no-auth-warning cluster nodes
[root@localhost ~]#redis-cli -a 123456 -h 10.0.0.8 --no-auth-warning cluster nodes
4e0eb42d503974e00962fa1d0214593720327d4a 10.0.0.28:6379@16379 master - 0 1642914346272 3 connected 10923-16383
67eb5b5226f01b6dde9145fe4063b1f4849fe921 10.0.0.48:6379@16379 slave 2f41d6b0a3b8957755b67b2dccfb764dee57f123 0 1642914339763 5 connected
2f41d6b0a3b8957755b67b2dccfb764dee57f123 10.0.0.18:6379@16379 master - 0 1642914344105 2 connected 5461-10922
7a4443e815ca6009d8e25e5f84d2f5feee3c007a 10.0.0.8:6379@16379 myself,slave 5308ffa51d731096d8c9e86d2f3976cb94fa7fa8 0 1642913710880 1 connected
128518070585eb054210b322b8a6f910cc2849f8 10.0.0.58:6379@16379 slave 4e0eb42d503974e00962fa1d0214593720327d4a 0 1642914345195 6 connected
5308ffa51d731096d8c9e86d2f3976cb94fa7fa8 10.0.0.38:6379@16379 master - 0 1642914341929 8 connected 0-5460

可以看出，master1恢复故障并没有重新获取master角色，而是以slave角色存在