solr排序OOM解决方法
问题lucene使用排序时会将被排序字段全部加入内存再进行排序，当多次使用不同字段进行排序时会造成OOM问题
解决方案
修改lucene源码在每次查询完成后将排序所使用的FieldCache里的缓存清空
分别需要修改两个工程
[*]lucene2.4
[*]solr1.3
Lucene2.4中要修改的类或接口有
[*]org.apache.lucene.index.IndexReader
添加成员变量
public String m_cacheKey = "cmhk_oom_bugfix";
[*]org.apache.lucene.search.Searcher
添加抽象方法
abstract public void release();
在它的实现类中除org.apache.lucene.search.IndexSearcher外的都加上一个这个抽象方法的空实现
[*]org.apache.lucene.search.IndexSearcher
添加方法
public void release() {
// 清除缓存数据
if (null != reader.m_cacheKey && !"".equals(reader.m_cacheKey)) {
if (null != FieldSortedHitQueue.Comparators) {
FieldSortedHitQueue.Comparators.ReleaseCache(reader.m_cacheKey);
}
FieldCache.DEFAULT.ReleaseFieldCache(reader.m_cacheKey);
ExtendedFieldCache.EXT_DEFAULT.ReleaseExtendedFieldCache(reader.m_cacheKey);
// 立即回收垃圾
// System.gc();
}
}
[*]org.apache.lucene.search.FieldCache
添加方法
void ReleaseFieldCache(String strCacheKey);
[*]org.apache.lucene.search.FieldCacheImpl
添加方法
public void ReleaseFieldCache(String strCacheKey) {
bytesCache.ReleaseCache(strCacheKey);
shortsCache.ReleaseCache(strCacheKey);
intsCache.ReleaseCache(strCacheKey);
floatsCache.ReleaseCache(strCacheKey);
stringsCache.ReleaseCache(strCacheKey);
stringsIndexCache.ReleaseCache(strCacheKey);
autoCache.ReleaseCache(strCacheKey);
customCache.ReleaseCache(strCacheKey);
}
[*]org.apache.lucene.search.FieldCacheImpl.Cache
添加方法
// 用于清除缓存
public void ReleaseCache(String strCacheKey) {
if (null != readerCache) {
synchronized (readerCache) {
if (null != readerCache.get(strCacheKey)) {
((Map) readerCache.get(strCacheKey)).clear();
}
}
}
}
修改org.apache.lucene.search.FieldCacheImpl.Cache.get(IndexReader, Object)方法
public Object get(IndexReader reader, Object key) throws IOException {
Map innerCache;
Object value;
synchronized (readerCache) {
//改键值为静态值
innerCache = (Map)readerCache.get(reader.m_cacheKey);
// innerCache = (Map) readerCache.get(reader);
if (innerCache == null) {
innerCache = new HashMap();
//改键值为静态值
readerCache.put(reader.m_cacheKey,innerCache);
//readerCache.put(reader, innerCache);
value = null;
} else {
value = innerCache.get(key);
}
if (value == null) {
value = new CreationPlaceholder();
innerCache.put(key, value);
}
}
if (value instanceof CreationPlaceholder) {
synchronized (value) {
CreationPlaceholder progress = (CreationPlaceholder) value;
if (progress.value == null) {
progress.value = createValue(reader, key);
synchronized (readerCache) {
innerCache.put(key, progress.value);
}
}
return progress.value;
}
}
return value;
}
}
[*]org.apache.lucene.search.ExtendedFieldCache
添加方法
//用于清除缓存
void ReleaseExtendedFieldCache(String strCacheKey);
[*]org.apache.lucene.search.ExtendedFieldCacheImpl
添加方法
// 用于清除缓存
public void ReleaseExtendedFieldCache(String strCacheKey) {
longsCache.ReleaseCache(strCacheKey);
doublesCache.ReleaseCache(strCacheKey);
autoCache.ReleaseCache(strCacheKey);
}
solr1.3中要修改的类或接口
[*]org.apache.solr.search.SolrIndexSearcher
添加方法
public void release(){
searcher.release();
}
[*]org.apache.solr.core.SolrCore
修改私有方法
private RefCounted<SolrIndexSearcher> newHolder(SolrIndexSearcher newSearcher) {
RefCounted<SolrIndexSearcher> holder = new RefCounted<SolrIndexSearcher>(newSearcher) {
public void close() {
try {
synchronized(searcherLock) {
// it's possible for someone to get a reference via the _searchers queue
// and increment the refcount while RefCounted.close() is being called.
// we check the refcount again to see if this has happened and abort the close.
// This relies on the RefCounted class allowing close() to be called every
// time the counter hits zero.
resource.release();//释放FieldCache中的内存
if (refcount.get() > 0) return;
_searchers.remove(this);
}
resource.close();
} catch (IOException e) {
log.severe("Error closing searcher:" + SolrException.toStr(e));
}
}
};
holder.incref(); // set ref count to 1 to account for this._searcher
return holder;
}
[*]org.apache.solr.util.RefCounted<Type>
修改方法org.apache.solr.util.RefCounted.decref()
public void decref() {
// if (refcount.decrementAndGet() == 0) {
close();
// }
}
页:
[1]