solr排序OOM解决方法

huhahapz

问题

　　lucene使用排序时会将被排序字段全部加入内存再进行排序，当多次使用不同字段进行排序时会造成OOM问题
解决方案

　　修改lucene源码在每次查询完成后将排序所使用的FieldCache里的缓存清空
分别需要修改两个工程

• lucene2.4
  
• solr1.3
  

Lucene2.4中要修改的类或接口有

• org.apache.lucene.index.IndexReader
  

添加成员变量
public String m_cacheKey = "cmhk_oom_bugfix";

• org.apache.lucene.search.Searcher
  

添加抽象方法
abstract public void release();

在它的实现类中除org.apache.lucene.search.IndexSearcher外的都加上一个这个抽象方法的空实现

• org.apache.lucene.search.IndexSearcher
  

添加方法
public void release() {
// 清除缓存数据
if (null != reader.m_cacheKey && !"".equals(reader.m_cacheKey)) {
if (null != FieldSortedHitQueue.Comparators) {
            FieldSortedHitQueue.Comparators.ReleaseCache(reader.m_cacheKey);
        }
        FieldCache.DEFAULT.ReleaseFieldCache(reader.m_cacheKey);
        ExtendedFieldCache.EXT_DEFAULT.ReleaseExtendedFieldCache(reader.m_cacheKey);
// 立即回收垃圾
// System.gc();
    }
}

• org.apache.lucene.search.FieldCache
  

添加方法
void ReleaseFieldCache(String strCacheKey);

• org.apache.lucene.search.FieldCacheImpl
  

添加方法
public void ReleaseFieldCache(String strCacheKey) {
        bytesCache.ReleaseCache(strCacheKey);
        shortsCache.ReleaseCache(strCacheKey);
        intsCache.ReleaseCache(strCacheKey);
        floatsCache.ReleaseCache(strCacheKey);
        stringsCache.ReleaseCache(strCacheKey);
        stringsIndexCache.ReleaseCache(strCacheKey);
        autoCache.ReleaseCache(strCacheKey);
        customCache.ReleaseCache(strCacheKey);
}

• org.apache.lucene.search.FieldCacheImpl.Cache
  

添加方法
// 用于清除缓存
public void ReleaseCache(String strCacheKey) {
if (null != readerCache) {
synchronized (readerCache) {
if (null != readerCache.get(strCacheKey)) {
                ((Map) readerCache.get(strCacheKey)).clear();
            }
        }
    }
}

修改org.apache.lucene.search.FieldCacheImpl.Cache.get(IndexReader, Object)方法
public Object get(IndexReader reader, Object key) throws IOException {
      Map innerCache;
      Object value;
synchronized (readerCache) {
//改键值为静态值
        innerCache = (Map)readerCache.get(reader.m_cacheKey);
// innerCache = (Map) readerCache.get(reader);
if (innerCache == null) {
          innerCache = new HashMap();
//改键值为静态值
          readerCache.put(reader.m_cacheKey,innerCache);
//readerCache.put(reader, innerCache);
          value = null;
        } else {
          value = innerCache.get(key);
        }
if (value == null) {
          value = new CreationPlaceholder();
          innerCache.put(key, value);
        }
      }
if (value instanceof CreationPlaceholder) {
synchronized (value) {
          CreationPlaceholder progress = (CreationPlaceholder) value;
if (progress.value == null) {
            progress.value = createValue(reader, key);
synchronized (readerCache) {
              innerCache.put(key, progress.value);
            }
          }
return progress.value;
        }
      }
return value;
    }
  }

• org.apache.lucene.search.ExtendedFieldCache
  

添加方法
//用于清除缓存
void ReleaseExtendedFieldCache(String strCacheKey);

• org.apache.lucene.search.ExtendedFieldCacheImpl
  

添加方法
// 用于清除缓存
public void ReleaseExtendedFieldCache(String strCacheKey) {
    longsCache.ReleaseCache(strCacheKey);
    doublesCache.ReleaseCache(strCacheKey);
    autoCache.ReleaseCache(strCacheKey);
}

solr1.3中要修改的类或接口

• org.apache.solr.search.SolrIndexSearcher
  

添加方法
public void release(){
      searcher.release();
}

• org.apache.solr.core.SolrCore
  

修改私有方法
private RefCounted<SolrIndexSearcher> newHolder(SolrIndexSearcher newSearcher) {
    RefCounted<SolrIndexSearcher> holder = new RefCounted<SolrIndexSearcher>(newSearcher) {
public void close() {
try {
synchronized(searcherLock) {
// it's possible for someone to get a reference via the _searchers queue
// and increment the refcount while RefCounted.close() is being called.
// we check the refcount again to see if this has happened and abort the close.
// This relies on the RefCounted class allowing close() to be called every
// time the counter hits zero.
            resource.release();//释放FieldCache中的内存
if (refcount.get() > 0) return;
            _searchers.remove(this);
          }
          resource.close();
        } catch (IOException e) {
          log.severe("Error closing searcher:" + SolrException.toStr(e));
        }
      }
    };
    holder.incref(); // set ref count to 1 to account for this._searcher
return holder;
  }

• org.apache.solr.util.RefCounted<Type>
  

修改方法org.apache.solr.util.RefCounted.decref()
public void decref() {
// if (refcount.decrementAndGet() == 0) {
  close();
// }
}