Solr Schema XML DTD校验

hti

     由于编辑solr的schema文件的过程中，时常会发生一些小错误，类似于，“true” 写成了“ture”
或者“multiValued”写成“multivalue”，虽然是手误但是往往会造成很诡异的错误，并且非常难排查错误。类似这样的笔误导致在生产环境中的bug，非常隐蔽不容易被发现，排除这样的错误经常要花上半天的时间或者更长。
   解决这个问题的办法是通过在编辑后执行文档Schema校验的方式来校验文档的正确行，校验方式有两种一种是DTD，另外一种方式是Schema,就是XSD（XML Schema Definition)这种方式对XML校验更加完善，可以对属性进行类型校验。
 
   简单起见，对schema.xml 利用DTD方式进行校验。以下是我针对solr的schema文件写DTD文件：
 

<!ELEMENT schema (types+,fields+,uniqueKey,defaultSearchField,similarity?,solrQueryParser,copyField*)>
<!ATTLIST schema name CDATA #REQUIRED>
<!ATTLIST schema version CDATA #REQUIRED>
<!ELEMENT types (fieldType+)>
<!ELEMENT fieldType (analyzer*)>
<!ATTLIST fieldType name CDATA #REQUIRED>
<!ATTLIST fieldType class CDATA #REQUIRED>
<!ATTLIST fieldType positionIncrementGap CDATA #IMPLIED>
<!ATTLIST fieldType sortMissingLast (true|false) "true">
<!ATTLIST fieldType omitNorms (true|false) "true">
<!ATTLIST fieldType precisionStep CDATA #IMPLIED>
<!ATTLIST fieldType autoGeneratePhraseQueries (true|false) #IMPLIED>
<!ATTLIST fieldType tokenized (true|false) #IMPLIED>
<!ATTLIST fieldType omitTermFreqAndPositions (true|false) #IMPLIED>

<!ELEMENT analyzer (tokenizer*,filter*)>
<!ATTLIST analyzer
type (index|query) #IMPLIED
class CDATA        #IMPLIED>
<!ELEMENT tokenizer EMPTY>
<!ATTLIST tokenizer
class CDATA #REQUIRED
group CDATA #IMPLIED
pattern CDATA #IMPLIED>
<!ELEMENT filter EMPTY>
<!ATTLIST filter class CDATA #REQUIRED>
<!ELEMENT uniqueKey (#PCDATA)>
<!ELEMENT defaultSearchField (#PCDATA)>
<!ELEMENT similarity (str*)>
<!ATTLIST similarity class CDATA #REQUIRED>
<!ELEMENT str (#PCDATA)>
<!ATTLIST str name CDATA #REQUIRED>

<!ELEMENT solrQueryParser EMPTY>
<!ATTLIST solrQueryParser defaultOperator (OR|AND) #REQUIRED>
<!ELEMENT fields (field|dynamicField)+>
<!ELEMENT field  EMPTY>
<!ATTLIST field  name CDATA #REQUIRED>
<!ATTLIST field  type CDATA #REQUIRED>
<!ATTLIST field  indexed (true|false) #IMPLIED>
<!ATTLIST field  stored (true|false) #IMPLIED>
<!ATTLIST field  multiValued (true|false) #IMPLIED>
<!ATTLIST field  required (true|false) #IMPLIED>
<!ATTLIST field  default CDATA #IMPLIED>
<!ELEMENT dynamicField EMPTY>
<!ATTLIST dynamicField  name CDATA #REQUIRED>
<!ATTLIST dynamicField  type CDATA #REQUIRED>
<!ATTLIST dynamicField  indexed (true|false) #IMPLIED>
<!ATTLIST dynamicField  stored (true|false) #IMPLIED>
<!ATTLIST dynamicField  multiValued (true|false) #IMPLIED>
<!ATTLIST dynamicField  required (true|false) #IMPLIED>
<!ELEMENT copyField EMPTY>
<!ATTLIST copyField  source CDATA #REQUIRED>
<!ATTLIST copyField  dest CDATA #REQUIRED>

 然后就可以用这个dtd来校验XML文件了，以下是一个schema XML文件样例：

<?xml version="1.0" ?>
<!DOCTYPE schema SYSTEM  "solrschema.dtd">
<schema name="example core" version="1.1">
<types>
<fieldType name="string" class="solr.TextField"
sortMissingLast="true"></fieldType>
<fieldType name="int" class="solr.TrieIntField"
precisionStep="0" positionIncrementGap="0" />
<fieldType name="tlong" class="solr.TrieLongField"
precisionStep="8" omitNorms="true" positionIncrementGap="0" />
<fieldType name="sint" class="solr.SortableIntField"
sortMissingLast="true" omitNorms="true" />
<fieldType name="slong" class="solr.SortableLongField"
sortMissingLast="true" omitNorms="true" />
<fieldType name="string_ws" class="solr.TextField"
positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.WhitespaceTokenizerFactory" />
</analyzer>
</fieldType>
<fieldType name="string_sd" class="solr.TextField"
sortMissingLast="true" omitNorms="true">
<analyzer type="index">
<tokenizer class="solr.StandardTokenizerFactory" />
<filter class="solr.ChineseFilterFactory" />
<filter class="solr.LowerCaseFilterFactory" />
</analyzer>
<analyzer type="query">
<tokenizer class="solr.StandardTokenizerFactory" />
<filter class="solr.ChineseFilterFactory" />
<filter class="solr.LowerCaseFilterFactory" />
</analyzer>
</fieldType>
<fieldType name="tint" class="solr.TrieIntField"
precisionStep="8" omitNorms="true" positionIncrementGap="0" />
<fieldType name="tfloat" class="solr.TrieFloatField"
precisionStep="8" omitNorms="true" positionIncrementGap="0" />
<fieldType name="sfloat" class="solr.SortableFloatField"
sortMissingLast="true" omitNorms="true" />
</types>
<fields>
<field name="id" type="slong" indexed="true" stored="true"
multiValued="false" required="true" />
<field name="userId" type="slong" indexed="true" stored="true"
multiValued="false" />
<field name="userNick" type="string" indexed="true" stored="true"
multiValued="false" />
<field name="title" type="string_sd" indexed="true" stored="true"
multiValued="false" />
<field name="contentKey" type="string" indexed="true" stored="true"
multiValued="false" />
<field name="coverPicpath" type="string" indexed="true" stored="true"
multiValued="false" />
<field name="itemId" type="string" indexed="true" stored="true"
multiValued="false" />
<field name="gmtCreated" type="slong" indexed="true" stored="true"
multiValued="false" />
<field name="gmtModified" type="slong" indexed="true" stored="true"
multiValued="false" />
<field name="tag" type="string_sd" indexed="true" stored="true"
multiValued="false" />
<field name="aScore" type="sfloat" indexed="true" stored="true"
multiValued="false" />
<field name="count" type="int" indexed="true" stored="true"
multiValued="false" />
<field name="feedId" type="string" stored="true" multiValued="false" />
<field name="summary" type="string_sd" indexed="true" stored="true"
multiValued="false" />
<field name="status" type="int" indexed="true" stored="true"
multiValued="false" />
<dynamicField name="*-t" type="text_com" stored="false"
indexed="true" />
</fields>
<uniqueKey>id</uniqueKey>
<defaultSearchField>id</defaultSearchField>
<similarity class="com.taobao.terminator.core.solrx.PayloadSimilarity">
<str name="func">max</str>
</similarity>
<solrQueryParser defaultOperator="OR" />
<copyField source="prov" dest="location" />
</schema>
 
写一个java的单元测试类来校验，XML的正确性：
 

import java.io.InputStream;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import junit.framework.Assert;
import junit.framework.TestCase;
import org.xml.sax.SAXException;
import org.xml.sax.SAXParseException;
import org.xml.sax.helpers.DefaultHandler;
/**
* @author 百岁（baisui@taobao.com）
* @date 2012-11-28
*/
public class TestSchemaDTDValidate extends TestCase {
private final static DocumentBuilderFactory factory = DocumentBuilderFactory
.newInstance();
static {
factory.setValidating(true);
}
public void testFile1() {
Assert.assertTrue("schema3.xml", XMLValidateDTD("schema3.xml"));
}
public boolean XMLValidateDTD(String file) {
MyHandler mh = new MyHandler();
InputStream inputStream = null;
try {
DocumentBuilder builder = factory.newDocumentBuilder();
builder.setErrorHandler(mh);
inputStream = this.getClass().getResourceAsStream("/" + file);
builder.parse(inputStream);
} catch (Exception e) {
Assert.assertFalse(e.getMessage(), true);
return false;
} finally {
try {
inputStream.close();
} catch (Throwable e) {
}
}
return mh.validate;
}
static class MyHandler extends DefaultHandler {
String errorMessage = null;
private boolean validate = true;
public void error(SAXParseException e) throws SAXException {
errorMessage = e.getMessage();
System.out.println(errorMessage + " linenumber:"
+ e.getLineNumber());
Assert.assertFalse(errorMessage, true);
validate = false;
}
public void fatalError(SAXParseException e) throws SAXException {
errorMessage = e.getMessage();
System.out.println(errorMessage);
validate = false;
Assert.assertFalse(errorMessage + " linenumber:"
+ e.getLineNumber(), true);
}
}
}
 
使用DTD来校验XML有以下不足：
 

• 子元素在排列顺序上有限制，是实际的xml文档中，定义的元素节点，必须要按照DTD中定义的顺序，哪怕这几个子元素在逻辑上并没有逻辑以来关系，比如，以上xml中<uniqueKey/>必须在<defaultSearchField/> 之前，如果不是这样的话校验XML过程中就会校验失败。
  
• 不能对节点属性的类型进行校验，比如,fieldType节点的precisionStep属性必须为数字，在实际中如果用户填写了非数字的字符作为的属性的话，校验也会通过的。这样就会发生潜在的错误。
  

虽然现在使用DTD有以上不足，但是相对以前完全没有校验机制来说已经进步一大截了，可以帮助我们规避掉很大一部分不必要的麻烦问题。如果想要弥补以上不足的话可以尝试一下XSD的校验机制。