Solr 学习（3） —-Solr 数据导入 <一>DIH简单使用

wooyun · 发表于 2015-7-16 11:58:21

　　转载自 http://martin3000.iteye.com/blog/1328833

　　使用DataImportHandler进行简单数据导入还是比较有效的，特别是DIH中针对简单的数据库表，可以把完全导入和增量导入合并成一个语句，非常方便。我的使用方式如下所示
　　1。配置schema
　　
　　

Xml代码 http://martin3000.iteye.com/javascripts/syntaxhighlighter/clipboard_new.swf?clipboard=%3CrequestHandler%20name%3D%22%2Fdataimport%22%20class%3D%22org.apache.solr.handler.dataimport.DataImportHandler%22%3E%0A%20%20%20%20%3Clst%20name%3D%22defaults%22%3E%0A%20%20%20%20%20%20%3Cstr%20name%3D%22config%22%3E%2Fhome%2Ftomcat%2Fbin%2Fsolr%2Fconf%2Fdata-config.xml%3C%2Fstr%3E%0A%20%20%20%20%3C%2Flst%3E%0A%20%20%3C%2FrequestHandler%3E

/home/tomcat/bin/solr/conf/data-config.xml

　　
　　3.让DIH周期性的运行
　　修改dataimport.properties文件，这个是自动生成的，同在solr/conf下，添加参数
　　interval 间隔时间单位分钟
　　syncEnabled=1 打开周期运行
　　params 其实就是具体调用的url，周期运行就是周期性的访问一个url
　　
　　

Java代码 http://martin3000.iteye.com/javascripts/syntaxhighlighter/clipboard_new.swf?clipboard=%23Wed%20Dec%2028%2009%3A29%3A42%20UTC%202011%0Aport%3D8983%0Ainterval%3D5%0Alast_index_time%3D2011-12-28%2009%5C%3A29%5C%3A26%0AsyncEnabled%3D1%0Awebapp%3Dsolr%0Aid.last_index_time%3D2011-12-28%2009%5C%3A29%5C%3A26%0Aserver%3D127.0.0.1%0Aparams%3D%2Fselect%3Fqt%5C%3D%2Fdataimport%26command%5C%3Dfull-import%26clean%5C%3Dfalse%26commit%5C%3Dtrue%26optimize%5C%3Dfalse

#Wed Dec 28 09:29:42 UTC 2011
port=8983
interval=5
last_index_time=2011-12-28 09\:29\:26
syncEnabled=1
webapp=solr
id.last_index_time=2011-12-28 09\:29\:26
server=127.0.0.1
params=/select?qt\=/dataimport&command\=full-import&clean\=false&commit\=true&optimize\=false

　　
　　到此还并不能周期运行，在solr的wiki中有一段实现这个功能的代码，但并没有加入到solr的发行包中，于是我们需要重新编译这段代码，打包放到webapp/solr/WEB-INF/lib中才行
　　

Xml代码 http://martin3000.iteye.com/javascripts/syntaxhighlighter/clipboard_new.swf?clipboard=%3Cweb-app%3E%0A%20%20%20%3Clistener%3E%0A%20%20%20%20%20%20%20%3Clistener-class%3Eorg.apache.solr.handler.dataimport.scheduler.ApplicationListener%3C%2Flistener-class%3E%0A%20%20%3C%2Flistener%3E%0A%20%20...%0A%3C%2Fweb-app%3E

org.apache.solr.handler.dataimport.scheduler.ApplicationListener
...

package org.apache.solr.handler.dataimport.scheduler;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Properties;
import org.apache.solr.core.SolrResourceLoader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SolrDataImportProperties {
private Properties properties;
public static final String SYNC_ENABLED = "syncEnabled";
public static final String SYNC_CORES = "syncCores";
public static final String SERVER = "server";
public static final String PORT = "port";
public static final String WEBAPP = "webapp";
public static final String PARAMS = "params";
public static final String INTERVAL = "interval";
private static final Logger logger = LoggerFactory.getLogger(SolrDataImportProperties.class);
public SolrDataImportProperties(){
// loadProperties(true);
}
public void loadProperties(boolean force){
try{
SolrResourceLoader loader = new SolrResourceLoader(null);
logger.info("Instance dir = " + loader.getInstanceDir());
String configDir = loader.getConfigDir();
configDir = SolrResourceLoader.normalizeDir(configDir);
if(force || properties == null){
properties = new Properties();
String dataImportPropertiesPath = configDir + "\\dataimport.properties";
FileInputStream fis = new FileInputStream(dataImportPropertiesPath);
properties.load(fis);
}
}catch(FileNotFoundException fnfe){
logger.error("Error locating DataImportScheduler dataimport.properties file", fnfe);
}catch(IOException ioe){
logger.error("Error reading DataImportScheduler dataimport.properties file", ioe);
}catch(Exception e){
logger.error("Error loading DataImportScheduler properties", e);
}
}
public String getProperty(String key){
return properties.getProperty(key);
}
}

package org.apache.solr.handler.dataimport.scheduler;
import java.util.Calendar;
import java.util.Date;
import java.util.Timer;
import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ApplicationListener implements ServletContextListener {
private static final Logger logger = LoggerFactory.getLogger(ApplicationListener.class);
@Override
public void contextDestroyed(ServletContextEvent servletContextEvent) {
ServletContext servletContext = servletContextEvent.getServletContext();
// get our timer from the context
Timer timer = (Timer)servletContext.getAttribute("timer");
// cancel all active tasks in the timers queue
if (timer != null)
timer.cancel();
// remove the timer from the context
servletContext.removeAttribute("timer");
}
@Override
public void contextInitialized(ServletContextEvent servletContextEvent) {
ServletContext servletContext = servletContextEvent.getServletContext();
try{
// create the timer and timer task objects
Timer timer = new Timer();
HTTPPostScheduler task = new HTTPPostScheduler(servletContext.getServletContextName(), timer);
// get our interval from HTTPPostScheduler
int interval = task.getIntervalInt();
// get a calendar to set the start time (first run)
Calendar calendar = Calendar.getInstance();
// set the first run to now + interval (to avoid fireing while the app/server is starting)
calendar.add(Calendar.MINUTE, interval);
Date startTime = calendar.getTime();
// schedule the task
timer.scheduleAtFixedRate(task, startTime, 1000 * 60 * interval);
// save the timer in context
servletContext.setAttribute("timer", timer);
} catch (Exception e) {
if(e.getMessage().endsWith("disabled")){
logger.info("Schedule disabled");
}else{
logger.error("Problem initializing the scheduled task: ", e);
}
}
}
}

package org.apache.solr.handler.dataimport.scheduler;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Timer;
import java.util.TimerTask;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class HTTPPostScheduler extends TimerTask {
private String syncEnabled;
private String[] syncCores;
private String server;
private String port;
private String webapp;
private String params;
private String interval;
private String cores;
private SolrDataImportProperties p;
private boolean singleCore;
private static final Logger logger = LoggerFactory.getLogger(HTTPPostScheduler.class);
public HTTPPostScheduler(String webAppName, Timer t) throws Exception{
//load properties from global dataimport.properties
p = new SolrDataImportProperties();
reloadParams();
fixParams(webAppName);
if(!syncEnabled.equals("1")) throw new Exception("Schedule disabled");
if(syncCores == null || (syncCores.length == 1 && syncCores[0].isEmpty())){
singleCore = true;
logger.info(" Single core identified in dataimport.properties");
}else{
singleCore = false;
logger.info(" Multiple cores identified in dataimport.properties. Sync active for: " + cores);
}
}
private void reloadParams(){
p.loadProperties(true);
syncEnabled = p.getProperty(SolrDataImportProperties.SYNC_ENABLED);
cores = p.getProperty(SolrDataImportProperties.SYNC_CORES);
server = p.getProperty(SolrDataImportProperties.SERVER);
port = p.getProperty(SolrDataImportProperties.PORT);
webapp = p.getProperty(SolrDataImportProperties.WEBAPP);
params = p.getProperty(SolrDataImportProperties.PARAMS);
interval = p.getProperty(SolrDataImportProperties.INTERVAL);
syncCores = cores != null ? cores.split(",") : null;
}
private void fixParams(String webAppName){
if(server == null || server.isEmpty()) server = "localhost";
if(port == null || port.isEmpty()) port = "8080";
if(webapp == null || webapp.isEmpty()) webapp = webAppName;
if(interval == null || interval.isEmpty() || getIntervalInt()

账号		自动登录	找回密码
密码			立即注册

zabbix3.4中文手册，官网完整COPY（2019042

最新rhel8官方手册三本PDF

winhex数据恢复教程（非常巨大，内容丰富）

KMSpico10.2.0 免费激活Win10/Office2016（

zabbix3.4.1安装部署+微信推送信息+大屏显

VMware vcenter+vSphere 6.5 U2共享

CentOS6.5下redis-3.2.6的安装与配置

[经验分享] Solr 学习（3） —-Solr 数据导入 <一>DIH简单使用

扫码加入运维网微信交流群