redis client高并发量死锁问题

xxqyzsc

在线上服务器跑一段时间之后发现大量如下错误,初步分析是redis实现的JedisPool的的配置有问题。
yunpeng@yunpeng-duitang:/work/xp$ cat aa.log | grep "java.lang.Object.wait(Native Method)" | wc -l
254
引用

"btpool0-1692" prio=10 tid=0x00002aaab54d8000 nid=0x2bf9 in Object.wait() [0x000000005bcf4000]
   java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:485)
at org.apache.commons.pool.impl.GenericObjectPool.borrowObject(GenericObjectPool.java:1104)
- locked <0x00000000c1ce6488> (a org.apache.commons.pool.impl.GenericObjectPool$Latch)
at redis.clients.util.Pool.getResource(Pool.java:20)
at org.springframework.data.redis.connection.jedis.JedisConnectionFactory.fetchJedisConnector(JedisConnectionFactory.java:94)


问题原因GenericObjectPool默认的maxActive只有8，参考：
http://commons.apache.org/pool/api-1.6/org/apache/commons/pool/impl/GenericObjectPool.html
spring 的代码结构：

class JedisConnectionFactory{
private JedisPool pool = null;
private JedisPoolConfig poolConfig = new JedisPoolConfig();
public void afterPropertiesSet() {
pool = new JedisPool(poolConfig, shardInfo.getHost(), shardInfo.getPort(), shardInfo.getTimeout(),shardInfo.getPassword());
}
protected Jedis fetchJedisConnector() {
return pool.getResource();
}
}
class Pool{  
private final GenericObjectPool internalPool;  
public Pool(final GenericObjectPool.Config poolConfig,  
PoolableObjectFactory factory) {  
this.internalPool = new GenericObjectPool(factory, poolConfig);  
}  
pool.getResource(){  
internalPool.borrowObject();  
}  
}  
class JedisPool extends Pool{
public JedisPool(final Config poolConfig, final String host, int port, int timeout, final String password,final int database) {
super(poolConfig, new JedisFactory(host, port, timeout, password, database));
}
}


写了一个测试代码重现了这个问题：

import org.apache.commons.pool.BasePoolableObjectFactory;
import org.apache.commons.pool.PoolableObjectFactory;
import org.apache.commons.pool.impl.GenericObjectPool;
import org.apache.commons.pool.impl.GenericObjectPool.Config;
public class GenericObjectPoolTest {
public static void main(String[] args) throws Exception {
Config config = new Config();
//config.setTestWhileIdle(true);
//config.setMinEvictableIdleTimeMillis(60000);
//config.setTimeBetweenEvictionRunsMillis(30000);
//config.setNumTestsPerEvictionRun(-1);
final GenericObjectPool internalPool = new GenericObjectPool(new ConnectionFactory(),config);
for (int x = 0;x<1000;++x){
Thread t  = new Thread(new Runnable() {
@Override
public void run() {
while(true){
try {
Object str = internalPool.borrowObject();
System.out.println(str);
} catch (Exception e) {
e.printStackTrace();
}
}
}
});
t.start();
}
}
static class ConnectionFactory extends BasePoolableObjectFactory implements PoolableObjectFactory{
@Override
public Object makeObject() throws Exception {
System.out.println(Thread.currentThread().getName()+" make....");
return "connection";
}

}
}

输出，只有8个object：
Thread-0 make....
Thread-1 make....
Thread-2 make....
Thread-3 make....
connection
connection
connection
Thread-2 make....
connection
Thread-2 make....
connection
Thread-2 make....
connection
Thread-1 make....
connection
connection

"Thread-999" prio=10 tid=0x00007f87dc6c0000 nid=0x213d in Object.wait() [0x00007f879a264000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        - waiting on <0x00000000e19f3a18> (a org.apache.commons.pool.impl.GenericObjectPool$Latch)
        at java.lang.Object.wait(Object.java:485)
        at org.apache.commons.pool.impl.GenericObjectPool.borrowObject(GenericObjectPool.java:1104)
        - locked <0x00000000e19f3a18> (a org.apache.commons.pool.impl.GenericObjectPool$Latch)
        at GenericObjectPoolTest$1.run(GenericObjectPoolTest.java:22)
        at java.lang.Thread.run(Thread.java:662)
"Thread-998" prio=10 tid=0x00007f87dc6be000 nid=0x213c in Object.wait() [0x00007f879a365000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        - waiting on <0x00000000e19f39e8> (a org.apache.commons.pool.impl.GenericObjectPool$Latch)
        at java.lang.Object.wait(Object.java:485)
        at org.apache.commons.pool.impl.GenericObjectPool.borrowObject(GenericObjectPool.java:1104)
        - locked <0x00000000e19f39e8> (a org.apache.commons.pool.impl.GenericObjectPool$Latch)
        at GenericObjectPoolTest$1.run(GenericObjectPoolTest.java:22)
        at java.lang.Thread.run(Thread.java:662)
"Thread-997" prio=10 tid=0x00007f87dc6bc000 nid=0x213b in Object.wait() [0x00007f879a466000]
   java.lang.Thread.State: WAITING (on object monitor)
        at java.lang.Object.wait(Native Method)
        - waiting on <0x00000000e19f39b8> (a org.apache.commons.pool.impl.GenericObjectPool$Latch)
        at java.lang.Object.wait(Object.java:485)
        at org.apache.commons.pool.impl.GenericObjectPool.borrowObject(GenericObjectPool.java:1104)
        - locked <0x00000000e19f39b8> (a org.apache.commons.pool.impl.GenericObjectPool$Latch)
        at GenericObjectPoolTest$1.run(GenericObjectPoolTest.java:22)
        at java.lang.Thread.run(Thread.java:662)