spring-data-redis读写分离

在对Redis进行性能优化时，一直想对Redis进行读写分离。但由于项目底层采用spring-data-redis对redis进行操作，参考spring官网却发现spring-data-redis目前(1.7.0.RELEASE)及以前的版本并不支持读写分离。

 一、源码分析

spring-data-redis中关于JedisConnectionFactory的配置如下：

<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xmlns:p="http://www.springframework.org/schema/p"
    xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-4.1.xsd">

    <bean id="redisSentinelConfiguration" class="org.springframework.data.redis.connection.RedisSentinelConfiguration">
        <property name="master">
            <bean class="org.springframework.data.redis.connection.RedisNode">
                <property name="name" value="mymaster"/>
                <constructor-arg name="host" value="${redis.master.host}"></constructor-arg>
                <constructor-arg name="port" value="${redis.master.port}"></constructor-arg>
            </bean>
        </property>
        <property name="sentinels">
            <set>
                <bean class="org.springframework.data.redis.connection.RedisNode">
                    <constructor-arg name="host" value="${redis.sentinel1.host}"></constructor-arg>
                    <constructor-arg name="port" value="${redis.sentinel1.port}"></constructor-arg>
                </bean>
                <bean class="org.springframework.data.redis.connection.RedisNode">
                    <constructor-arg name="host" value="${redis.sentinel2.host}"></constructor-arg>
                    <constructor-arg name="port" value="${redis.sentinel2.port}"></constructor-arg>
                </bean>
                <bean class="org.springframework.data.redis.connection.RedisNode">
                    <constructor-arg name="host" value="${redis.sentinel3.host}"></constructor-arg>
                    <constructor-arg name="port" value="${redis.sentinel3.port}"></constructor-arg>
                </bean>
            </set>
        </property>
    </bean>

    <!-- 连接池配置 -->
    <bean id="jedisPoolConfig" class="redis.clients.jedis.JedisPoolConfig">
        <property name="maxTotal" value="${redis.pool.maxActive}" />
        <property name="maxIdle" value="${redis.pool.maxIdle}" />
        <!-- <property name="maxWait" value="${redis.pool.maxWait}" /> -->
        <property name="testOnBorrow" value="${redis.pool.testOnBorrow}" />
        <property name="testOnReturn" value="${redis.pool.testOnReturn}" />
    </bean>

    <bean id="jedisConnectionFactory"
          class="org.springframework.data.redis.connection.jedis.JedisConnectionFactory">
        <property name="poolConfig" ref="jedisPoolConfig"></property>
        <constructor-arg ref="redisSentinelConfiguration"/>
    </bean>

    <bean id="stringRedisTemplate"
          class="org.springframework.data.redis.core.StringRedisTemplate"
          p:connection-factory-ref="jedisConnectionFactory"/>
</beans>

查看JedisConnectionFactory源码发现pool是Pool<Jedis>，而不是Pool<ShardedJedis>。因此我猜目前spring data redis是做不了读写分离的，stringRedisTemplate读写操作都是在master上。

二、读写分离改造

参考sentinel的主备选举机制对spring-data-redis的相关配置进行如下改造：

读写分离原理如下所示：

(1)Spring配置

<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xmlns:p="http://www.springframework.org/schema/p"
    xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-4.1.xsd">

       <context:property-placeholder  location="classpath:redis/redis.properties" ignore-unresolvable="true" />

       <bean id="poolConfig" class="redis.client.jedis.JedisPoolConfig">
            <property name="maxIdle" value="${redis.maxIdle}" />
            <property name="maxTotal" value="${redis.maxTotal}" />
            <property name="maxWaitMillis" value="${redis.maxWait}" />
            <property name="testOnBuorrow" value="${redis.testOnBorrow}" />
        </bean>

    <bean id="redisSentinelConfiguration" class="org.springframework.data.redis.connection.RedisSentinelConfiguration">
     <constructor-arg index="">
          <bean class="org.springframework.core.env.MapPropertySource">
            <constructor-arg index="" value="RedisSentinelConffiguration" />
            <constructor-arg index="">
               <map>
                <entry key="spring.redis.sentinel.master" value="${redis.sentinel.master}"></entry>
                <entry key="spring.redis.sentinel.nodes" value="${redis.sentinel.nodes"}> </entry>
                </map>
             </constructor-arg>
             </bean>
            </constructor-arg>
    </bean>

    <bean id="connectionFactory" class="com.test.data.redis.connection.TWJedisConnectionFactory">
       <constructor-arg index="" ref="redisSentinelConfiguration" />
       <constructor-arg index="" ref="poolConfig" />
       <property name="password" value="${redis.pass}" />
       <property name="databse" value="" />
    </bean>

    <bean id="readOnlyConnectionFactory" class="com.test.data.redis.connection.TWReadOnlyJedisConnectionFactory">
       <constructor-arg index="" ref="redisSentinelConfiguration" />
       <constructor-arg index="" ref="poolConfig" />
       <property name="password" value="${redis.pass}" />
       <property name="databse" value="" />
    </bean>

    <bean id="redisTemplate" class="com.test.data.redis.core.TWRedisTemplate"
       <property name="connectionFactory" ref="connectionFactory"/>
       <property name="readOnlyConnectionFactory" ref="readOnlyConnectionFactory" />
     </bean>
</beans>

(2)TWJedisConnectionFactory

public class TWJedisConnectionFactory extends JedisConnectionFactory {

  public TWJedisConnectionFactory() {
     super();
  }

  public TWJedisConnectionFactory(JedisShardInfo shardInfo) {
     super(shardInfo);
  }

  public TWJedisConnectionFactory(JedisPoolConfig poolConfig){
     this((RedisSentinelConfiguration)null,poolConfig);
  }

  public TWJedisConnectionFactory(RedisSentinelConfiguration sentinelConfig){
     this(sentinelConfig,null);
  }

  public TWJedisConnectionFactory(RedisSentinelConfiguration sentinelConfig,JedisPoolConfig poolConfig){
     super(sentinelConfig,poolConfig);
  }

  public TWJedisConnectionFactory(RedisClusterConfiguration clusterConfig){
     super(clusterConfig);
  }

  public TWJedisConnectionFactory(RedisClusterConfiguration clusterConfig,JedisPoolConfig poolConfig){
     super(clusterConfig,poolConfig);
  }

  @Override
  public void afterPropertiesSet() {
   try {
    super.afterPropertiesSet();
   }catch(Exception e) {
   }
  }
}

(3)TWReadOnlyJedisConnectionFactory

public class TWReadOnlyJedisConnectionFactory extends JedisConnectionFactory {

  private static final Method GET_TIMEOUT_METHOD;

  static {
     Method getTimeoutMethodCandidate = ReflectionUtils.findMethod(JedisShardInfo.class,"getTimeout");
     if(null == getTimeoutMethodCandidate) {
       getTimeoutMethodCandidate = ReflectionUtils.findMethod(JedisShardInfo.class,"getTimeout");
     }
     GET_TIMEOUT_METHOD=getTimeoutMethodCandidate;
  }

  public TWReadOnlyJedisConnectionFactory() {
     super();
  }

  public TWReadOnlyJedisConnectionFactory(JedisShardInfo shardInfo) {
     super(shardInfo);
  }

  public TWReadOnlyJedisConnectionFactory(JedisPoolConfig poolConfig){
     this((RedisSentinelConfiguration)null,poolConfig);
  }

  public TWReadOnlyJedisConnectionFactory(RedisSentinelConfiguration sentinelConfig){
     this(sentinelConfig,null);
  }

  public TWReadOnlyJedisConnectionFactory(RedisSentinelConfiguration sentinelConfig,JedisPoolConfig poolConfig){
     super(sentinelConfig,poolConfig);
  }

  public TWReadOnlyJedisConnectionFactory(RedisClusterConfiguration clusterConfig){
     super(clusterConfig);
  }

  public TWJedisConnectionFactory(RedisClusterConfiguration clusterConfig,JedisPoolConfig poolConfig){
     super(clusterConfig,poolConfig);
  }

  @Override
  public void afterPropertiesSet() {
   try {
    super.afterPropertiesSet();
   }catch(Exception e) {
   }
  }

  protected Pool<Jedis> createRedisSentinelPool(RedisSentinelConfiguration config){
    return new JedisSentinelSlavePool(config.getMaster().getName(),
           convertToJedisSentinelSet(config.getSentinels()),
           getPoolConfig()!=null?getPoolConfig():new JedisPoolConfig(),
           getTimeOutFrom(getShardInfo()),
           getShardInfo().getPassword());
  }

 private Set<String> convertToJedisSentinelSet(Collection<RedisNode> nodes) {
  if(CollectionUtils.isEmpty(nodes)) {
    return Collections.emptySet();
  }
  Set<String> convertedNodes = new LinkedHashSet<String>(nodes.size());
  for(RedisNode node : nodes)
   {
     convertedNodes.add(node.asString());
   }
  return convertedNodes;
 }

 private int getTimeOutFrom(JedisShardInfo shardInfo){
  return (Integer) ReflectionUtils.invokeMethod(GET_TIMEOUT_METHOD,shardInfo);
 }

}

(4)TWRedisTemplate

public class TWRedisTemplate extends RedisTemplate {
  public static enum Operation {read,write};

  private static ThreadLocal<Operation> threadLocal = new ThreadLocal<TWRedisTemplate.Operation>();

  private RedisConnectionFactory readOnlyConnectionFactory;  

  public void setReadOnlyConnectionFactory(RedisConnectionFactory readOnlyConnectionFactory) {
    this.readOnlyConnectionFactory = readOnlyConnectionFactory;
  }

  public TWRedisTemplate() {
  }

  @Override
  public void afterPropertiesSet() {
   try {
    super.afterPropertiesSet();
   }catch(Exception e) {
   }
  }

  public RedisConnectionFactory getConnectionFactory() {
    Operation operation = threadLocal.get();
    if(operation!=null && operation==Operation.read && readOnlyConnectionFactory!=null) {
      return readOnlyConnectionFactory;
    }
    return super.getConnectionFactory();
  }

 public void setOperation(Operation operation) {
   threadLocal.set(operation);
 }
}

(5)JedisReadOnlyPool

public class JedisReadOnlyPool extends Pool<Jedis> {
  protected GenericObjectPoolConfig poolConfig;
  protected int connectionTimeout = Protocol.DEFAULT_TIMEOUT;
  protected int soTimeout = Protocol.DEFAULT_TIMEOUT;
  protected String password;
  protected int database = Protocol.DEFAULT_TIMEOUT;
  private volatile JedisFactory factory;
  private volatile HostAndPort currentHostMaster;

  public JedisReadOnlyPool(final HostAndPort master, final GenericObjectPoolConfig poolConfig,
                           final int connectionTimeOut, final int soTimeout, final String password, final int databse,
                           final String clientName) {
         this.poolConfig = poolConfig;
         this.connectionTimeout = connectionTimeout;
         this.soTimeout = soTimeout;
         this.password = password;
         this.database = database;
         initPool(master);
  }
  public void destory() {
   super.destroy();
  }

  public void initPool(HostAndPort master) {
    if(!master.equals(currentHostMaster) {
       currentHostMaster = master;
       if(factory == null) {
         factory = new JedisFactory(currentHostMaster.getHost(),
                                    currentHostMaster.getPort(),
                                    connectionTimeout,
                                    soTimeout,
                                    password,
                                    database,
                                    null);
        initPool(poolConfig,factory);
    } else {
        factory.setHostAndPort(currentHostMaster);
        internalPool.clear();
    }
    log.info("Created JedisPool to master at" + master);
   }
  }

 private HostAndPort toHostAndPort(List<String> getMasterAddByNameResult) {
   String host = getMasterAddrByNameResult.get();
   int port = Integer.parseInt(getMasterAddrByNameResult.get());
   return new HostAndPort(host,port);
 }

 @Override
 public Jedis getResource() {
   while(true) {
      Jedis jedis = super.getResource();
      jedis.setDataSource(this);
      final HostAndPort connection = new HostAndPort(jedis.getClient().getHost(),jedis.getClient().getPort());
      if(currentHostMaster.equals(connection)) {
        return jedis;
      } else {
        returnBrokenResource(jedis);
      }
   }
 }

 @Override
 @Deprecated
 public void returnBrokenResource(final Jedis resource) {
   if(resource!=null) {
     returnBrokenResourceObject(resource);
   }
 }

 @Override
 @Deprecated
 public void returnResource(final Jedis resource) {
  if(resource !=null ){
   resource.resetState();
   returnResourceObject(resource);
  }
 }

}

(6)JedisSentinelSlavesPool

(7)RedisCacheService

在具体使用缓存服务时，在读、写缓存时分别加上其类型

...
@Autowired
private TWRedisTemplate twRedisTemplate;

public List<String> getCachedByPredis(final String prefix) {
  twRedisTemplate.setOperation(Operation.read);
  ...
  finally {
    destory();
  }
}

public void hset(xxx) {
  twRedisTemplate.setOperation(Operation.write);
  ...
  finally {
    destory();
  }

}

/**释放资源**/
private void destroy() {
  twRedisTemplate.setOperation(null);
}