python 2个html 解析器的比较 lxml.html 和 libxml2dom
lxml.html 和 libxml2dom 都很强大libxml2dom 和 lxml.html 都是基于 libxml的python包装，性能也都没有什么差别
lxml.html 使用起来比较方便，但是api 不标准
libxml2dom使用标准的api 用起来会比较舒服 和javascript的api一样
#coding:utf-8
test_html="""
<html>
<body>
ddd
<p>
p1
</html>
<html>
<body><p>p2
fff<p>
p3
<i>iiii</i>
xxx</p>
<br>
d<b>bb</b>
</p>
"""
def test_lxml():
import lxml.html
doc=lxml.html.fromstring(test_html)
#print dir(doc)
#print doc.text
for p in doc.xpath('//p'):
#print '*', p.text
#print dir(p )
print 'p========',id(p)
print lxml.html.tostring(p)
print 'getchildren'
for i in p.getchildren():
print '#',i
print 'itertext'
for i in p.itertext():
print '$',type(i),i
print 'iter'
for i in p.iter():
print '$',type(i),i
print 'items'
for i in p.items():
print '$',type(i),i
def test_xml2dom():
import libxml2dom
#help(libxml2dom)
doc = libxml2dom.parseString(test_html, html=1)
for p in doc.xpath('//p'):
print 'p========='
print p,type(p)
#print dir(p)
print p.childNodes
for n in p.childNodes:
#print dir(n)
print n.textContent,n.nodeName,n.nodeType
def test_speed():
import lxml.html
import libxml2dom
import time
def load_lxml(html):
doc=lxml.html.fromstring(html)
return len(doc.xpath('//p'))
def load_dom(html):
doc=libxml2dom.parseString(html)
return len(doc.xpath('//p'))
def test_fun(f):
t1=time.time()
s=0
for i in range(10000):
s+=load_lxml(test_html)
print f, time.time()-t1,s
test_fun(load_lxml)
test_fun(load_dom)
test_fun(load_lxml)
test_fun(load_dom)
if __name__=='__main__':
test_xml2dom()
test_lxml()
test_speed()
主要测试下速度 和 比较不正规的html解析
都比较不错，性能也相差不大
执行结果
引用
p=========
<libxml2dom.Node object at 0x8f1cf2c> <class 'libxml2dom.Node'>
[<libxml2dom.Node object at 0x8f4a36c>]
p1
text 3
p=========
<libxml2dom.Node object at 0x8f1cf8c> <class 'libxml2dom.Node'>
[<libxml2dom.Node object at 0x8f4a38c>]
p2
fff text 3
p=========
<libxml2dom.Node object at 0x8f4a32c> <class 'libxml2dom.Node'>
[<libxml2dom.Node object at 0x8f4a3ac>, <libxml2dom.Node object at 0x8f4a34c>, <libxml2dom.Node object at 0x8f4a3cc>]
p3
text 3
iiii i 1
xxx text 3
p======== 150665036
<p>
p1
</p>
getchildren
itertext
$ <type 'str'>
p1
iter
$ <class 'lxml.html.HtmlElement'> <Element p at 8faf74c>
items
p======== 150666476
<p>p2
fff</p>
getchildren
itertext
$ <type 'str'> p2
fff
iter
$ <class 'lxml.html.HtmlElement'> <Element p at 8fafcec>
items
p======== 150666620
<p>
p3
<i>iiii</i>
xxx</p>
getchildren
# <Element i at 8faf56c>
itertext
$ <type 'str'>
p3
$ <type 'str'> iiii
$ <type 'str'>
xxx
iter
$ <class 'lxml.html.HtmlElement'> <Element p at 8fafd7c>
$ <class 'lxml.html.HtmlElement'> <Element i at 90e6ecc>
items
<function load_lxml at 0x90f6e2c> 1.59452700615 30000
<function load_dom at 0x90f6e64> 1.68855881691 30000
<function load_lxml at 0x90f6e2c> 1.71267414093 30000
<function load_dom at 0x90f6e64> 1.74115109444 30000
以后还是倾向于使用 libxml2dom这个模块了
－－－－－－－－－－－－－－
附上libxml2dom的api
－－－－－－
Help on package libxml2dom:
NAME
libxml2dom - DOM wrapper around libxml2, specifically the libxml2mod Python extension module.
FILE
/usr/local/lib/python2.6/dist-packages/libxml2dom/__init__.py
DESCRIPTION
Copyright (C) 2003, 2004, 2005, 2006, 2007 Paul Boddie <paul@boddie.org.uk>
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU Lesser General Public License as published by the Free
Software Foundation; either version 3 of the License, or (at your option) any
later version.
This program is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE.See the GNU Lesser General Public License for more
details.
You should have received a copy of the GNU Lesser General Public License along
with this program.If not, see <http://www.gnu.org/licenses/>.
PACKAGE CONTENTS
events
macrolib (package)
soap
svg
xmlrpc
xmpp
CLASSES
__builtin__.list(__builtin__.object)
NodeList
__builtin__.object
DocumentType
Implementation
NamedNodeMap
NamedNodeMapIterator
Node
Attribute
Document(_Document, Node)
_Document
Document(_Document, Node)
class Attribute(Node)
|A class providing attribute access.
|
|Method resolution order:
| Attribute
| Node
| __builtin__.object
|
|Methods defined here:
|
|__init__(self, node, impl, ownerDocument=None, ownerElement=None)
|
|----------------------------------------------------------------------
|Data descriptors defined here:
|
|parentNode
|
|----------------------------------------------------------------------
|Methods inherited from Node:
|
|__eq__(self, other)
|
|__hash__(self)
|
|__ne__(self, other)
|
|appendChild(self, tmp)
|
|as_native_node(self)
|
|cloneNode(self, deep)
|
|createAttribute(self, name)
|
|createAttributeNS(self, ns, name)
|
|createCDATASection(self, value)
|
|createComment(self, value)
|
|createElement(self, name)
|
|createElementNS(self, ns, name)
|
|createTextNode(self, value)
|
|getAttribute(self, name)
|
|getAttributeNS(self, ns, localName)
|
|getAttributeNode(self, localName)
|
|getAttributeNodeNS(self, ns, localName)
|
|getElementById(self, identifier)
|
|getElementsByTagName(self, tagName)
|
|getElementsByTagNameNS(self, namespaceURI, localName)
|
|hasAttribute(self, name)
|
|hasAttributeNS(self, ns, localName)
|
|importNode(self, node, deep)
|
|insertBefore(self, tmp, oldNode)
|
|isSameNode(self, other)
|
|normalize(self)
|
|removeAttribute(self, name)
|
|removeAttributeNS(self, ns, localName)
|
|removeChild(self, tmp)
|
|replaceChild(self, tmp, oldNode)
|
|setAttribute(self, name, value)
|
|setAttributeNS(self, ns, name, value)
|
|setAttributeNode(self, node)
|
|setAttributeNodeNS(self, node)
|
|toFile(self, f, encoding=None, prettyprint=0)
|
|toStream(self, stream, encoding=None, prettyprint=0)
|
|toString(self, encoding=None, prettyprint=0)
|
|xpath(self, expr, variables=None, namespaces=None)
|
|----------------------------------------------------------------------
|Data descriptors inherited from Node:
|
|__dict__
| dictionary for instance variables (if defined)
|
|__weakref__
| list of weak references to the object (if defined)
|
|attributes
|
|childNodes
|
|data
|
|doctype
|
|firstChild
|
|lastChild
|
|localName
|
|name
|
|namespaceURI
|
|nextSibling
|
|nodeName
|
|nodeType
|
|nodeValue
|
|prefix
|
|previousSibling
|
|publicId
|
|systemId
|
|tagName
|
|textContent
|
|value
|
|----------------------------------------------------------------------
|Data and other attributes inherited from Node:
|
|ATTRIBUTE_NODE = 2
|
|COMMENT_NODE = 8
|
|DOCUMENT_NODE = 9
|
|DOCUMENT_TYPE_NODE = 10
|
|ELEMENT_NODE = 1
|
|ENTITY_NODE = 6
|
|ENTITY_REFERENCE_NODE = 5
|
|NOTATION_NODE = 12
|
|PROCESSING_INSTRUCTION_NODE = 7
|
|TEXT_NODE = 3
|
|entities = {}
|
|notations = {}
class Document(_Document, Node)
|A generic document class. Specialised document classes should inherit from
|the _Document class and their own variation of Node.
|
|Method resolution order:
| Document
| _Document
| Node
| __builtin__.object
|
|Methods inherited from _Document:
|
|__del__(self)
|
|__init__(self, node, impl)
|
|----------------------------------------------------------------------
|Data descriptors inherited from _Document:
|
|documentElement
|
|ownerDocument
|
|----------------------------------------------------------------------
|Methods inherited from Node:
|
|__eq__(self, other)
|
|__hash__(self)
|
|__ne__(self, other)
|
|appendChild(self, tmp)
|
|as_native_node(self)
|
|cloneNode(self, deep)
|
|createAttribute(self, name)
|
|createAttributeNS(self, ns, name)
|
|createCDATASection(self, value)
|
|createComment(self, value)
|
|createElement(self, name)
|
|createElementNS(self, ns, name)
|
|createTextNode(self, value)
|
|getAttribute(self, name)
|
|getAttributeNS(self, ns, localName)
|
|getAttributeNode(self, localName)
|
|getAttributeNodeNS(self, ns, localName)
|
|getElementById(self, identifier)
|
|getElementsByTagName(self, tagName)
|
|getElementsByTagNameNS(self, namespaceURI, localName)
|
|hasAttribute(self, name)
|
|hasAttributeNS(self, ns, localName)
|
|importNode(self, node, deep)
|
|insertBefore(self, tmp, oldNode)
|
|isSameNode(self, other)
|
|normalize(self)
|
|removeAttribute(self, name)
|
|removeAttributeNS(self, ns, localName)
|
|removeChild(self, tmp)
|
|replaceChild(self, tmp, oldNode)
|
|setAttribute(self, name, value)
|
|setAttributeNS(self, ns, name, value)
|
|setAttributeNode(self, node)
|
|setAttributeNodeNS(self, node)
|
|toFile(self, f, encoding=None, prettyprint=0)
|
|toStream(self, stream, encoding=None, prettyprint=0)
|
|toString(self, encoding=None, prettyprint=0)
|
|xpath(self, expr, variables=None, namespaces=None)
|
|----------------------------------------------------------------------
|Data descriptors inherited from Node:
|
|__dict__
| dictionary for instance variables (if defined)
|
|__weakref__
| list of weak references to the object (if defined)
|
|attributes
|
|childNodes
|
|data
|
|doctype
|
|firstChild
|
|lastChild
|
|localName
|
|name
|
|namespaceURI
|
|nextSibling
|
|nodeName
|
|nodeType
|
|nodeValue
|
|parentNode
|
|prefix
|
|previousSibling
|
|publicId
|
|systemId
|
|tagName
|
|textContent
|
|value
|
|----------------------------------------------------------------------
|Data and other attributes inherited from Node:
|
|ATTRIBUTE_NODE = 2
|
|COMMENT_NODE = 8
|
|DOCUMENT_NODE = 9
|
|DOCUMENT_TYPE_NODE = 10
|
|ELEMENT_NODE = 1
|
|ENTITY_NODE = 6
|
|ENTITY_REFERENCE_NODE = 5
|
|NOTATION_NODE = 12
|
|PROCESSING_INSTRUCTION_NODE = 7
|
|TEXT_NODE = 3
|
|entities = {}
|
|notations = {}
class DocumentType(__builtin__.object)
|A class providing a container for document type information.
|
|Methods defined here:
|
|__init__(self, localName, publicId, systemId)
|
|----------------------------------------------------------------------
|Data descriptors defined here:
|
|__dict__
| dictionary for instance variables (if defined)
|
|__weakref__
| list of weak references to the object (if defined)
class Implementation(__builtin__.object)
|Contains an abstraction over the DOM implementation.
|
|Methods defined here:
|
|adoptDocument(self, node)
|
|createDocument(self, namespaceURI, localName, doctype)
|
|createDocumentType(self, localName, publicId, systemId)
|
|get_node(self, _node, context_node)
|
|get_node_or_none(self, _node, context_node)
|
|----------------------------------------------------------------------
|Data descriptors defined here:
|
|__dict__
| dictionary for instance variables (if defined)
|
|__weakref__
| list of weak references to the object (if defined)
class NamedNodeMap(__builtin__.object)
|A wrapper around Node objects providing DOM and dictionary convenience
|methods.
|
|Methods defined here:
|
|__delitem__(self, name)
|
|__getitem__(self, name)
|
|__init__(self, node, impl)
|
|__iter__(self)
|
|__repr__(self)
|
|__setitem__(self, name, node)
|
|__str__(self)
|
|getNamedItem(self, name)
|
|getNamedItemNS(self, ns, localName)
|
|items(self)
|
|keys(self)
|
|removeNamedItem(self, name)
|
|removeNamedItemNS(self, ns, localName)
|
|setNamedItem(self, node)
|
|setNamedItemNS(self, node)
|
|values(self)
|
|----------------------------------------------------------------------
|Data descriptors defined here:
|
|__dict__
| dictionary for instance variables (if defined)
|
|__weakref__
| list of weak references to the object (if defined)
|
|length
class NamedNodeMapIterator(__builtin__.object)
|An iterator over a NamedNodeMap.
|
|Methods defined here:
|
|__init__(self, nodemap)
|
|next(self)
|
|----------------------------------------------------------------------
|Data descriptors defined here:
|
|__dict__
| dictionary for instance variables (if defined)
|
|__weakref__
| list of weak references to the object (if defined)
class Node(__builtin__.object)
|A DOM-style wrapper around libxml2mod objects.
|
|Methods defined here:
|
|__eq__(self, other)
|
|__hash__(self)
|
|__init__(self, node, impl=None, ownerDocument=None)
|
|__ne__(self, other)
|
|appendChild(self, tmp)
|
|as_native_node(self)
|
|cloneNode(self, deep)
|
|createAttribute(self, name)
|
|createAttributeNS(self, ns, name)
|
|createCDATASection(self, value)
|
|createComment(self, value)
|
|createElement(self, name)
|
|createElementNS(self, ns, name)
|
|createTextNode(self, value)
|
|getAttribute(self, name)
|
|getAttributeNS(self, ns, localName)
|
|getAttributeNode(self, localName)
|
|getAttributeNodeNS(self, ns, localName)
|
|getElementById(self, identifier)
|
|getElementsByTagName(self, tagName)
|
|getElementsByTagNameNS(self, namespaceURI, localName)
|
|hasAttribute(self, name)
|
|hasAttributeNS(self, ns, localName)
|
|importNode(self, node, deep)
|
|insertBefore(self, tmp, oldNode)
|
|isSameNode(self, other)
|
|normalize(self)
|
|removeAttribute(self, name)
|
|removeAttributeNS(self, ns, localName)
|
|removeChild(self, tmp)
|
|replaceChild(self, tmp, oldNode)
|
|setAttribute(self, name, value)
|
|setAttributeNS(self, ns, name, value)
|
|setAttributeNode(self, node)
|
|setAttributeNodeNS(self, node)
|
|toFile(self, f, encoding=None, prettyprint=0)
|
|toStream(self, stream, encoding=None, prettyprint=0)
|
|toString(self, encoding=None, prettyprint=0)
|
|xpath(self, expr, variables=None, namespaces=None)
|
|----------------------------------------------------------------------
|Data descriptors defined here:
|
|__dict__
| dictionary for instance variables (if defined)
|
|__weakref__
| list of weak references to the object (if defined)
|
|attributes
|
|childNodes
|
|data
|
|doctype
|
|firstChild
|
|lastChild
|
|localName
|
|name
|
|namespaceURI
|
|nextSibling
|
|nodeName
|
|nodeType
|
|nodeValue
|
|parentNode
|
|prefix
|
|previousSibling
|
|publicId
|
|systemId
|
|tagName
|
|textContent
|
|value
|
|----------------------------------------------------------------------
|Data and other attributes defined here:
|
|ATTRIBUTE_NODE = 2
|
|COMMENT_NODE = 8
|
|DOCUMENT_NODE = 9
|
|DOCUMENT_TYPE_NODE = 10
|
|ELEMENT_NODE = 1
|
|ENTITY_NODE = 6
|
|ENTITY_REFERENCE_NODE = 5
|
|NOTATION_NODE = 12
|
|PROCESSING_INSTRUCTION_NODE = 7
|
|TEXT_NODE = 3
|
|entities = {}
|
|notations = {}
class NodeList(__builtin__.list)
|A wrapper around node lists.
|
|Method resolution order:
| NodeList
| __builtin__.list
| __builtin__.object
|
|Methods defined here:
|
|item(self, index)
|
|----------------------------------------------------------------------
|Data descriptors defined here:
|
|__dict__
| dictionary for instance variables (if defined)
|
|__weakref__
| list of weak references to the object (if defined)
|
|length
|
|----------------------------------------------------------------------
|Methods inherited from __builtin__.list:
|
|__add__(...)
| x.__add__(y) <==> x+y
|
|__contains__(...)
| x.__contains__(y) <==> y in x
|
|__delitem__(...)
| x.__delitem__(y) <==> del x
|
|__delslice__(...)
| x.__delslice__(i, j) <==> del x
|
| Use of negative indices is not supported.
|
|__eq__(...)
| x.__eq__(y) <==> x==y
|
|__ge__(...)
| x.__ge__(y) <==> x>=y
|
|__getattribute__(...)
| x.__getattribute__('name') <==> x.name
|
|__getitem__(...)
| x.__getitem__(y) <==> x
|
|__getslice__(...)
| x.__getslice__(i, j) <==> x
|
| Use of negative indices is not supported.
|
|__gt__(...)
| x.__gt__(y) <==> x>y
|
|__iadd__(...)
| x.__iadd__(y) <==> x+=y
|
|__imul__(...)
| x.__imul__(y) <==> x*=y
|
|__init__(...)
| x.__init__(...) initializes x; see x.__class__.__doc__ for signature
|
|__iter__(...)
| x.__iter__() <==> iter(x)
|
|__le__(...)
| x.__le__(y) <==> x<=y
|
|__len__(...)
| x.__len__() <==> len(x)
|
|__lt__(...)
| x.__lt__(y) <==> x<y
|
|__mul__(...)
| x.__mul__(n) <==> x*n
|
|__ne__(...)
| x.__ne__(y) <==> x!=y
|
|__repr__(...)
| x.__repr__() <==> repr(x)
|
|__reversed__(...)
| L.__reversed__() -- return a reverse iterator over the list
|
|__rmul__(...)
| x.__rmul__(n) <==> n*x
|
|__setitem__(...)
| x.__setitem__(i, y) <==> x=y
|
|__setslice__(...)
| x.__setslice__(i, j, y) <==> x=y
|
| Useof negative indices is not supported.
|
|__sizeof__(...)
| L.__sizeof__() -- size of L in memory, in bytes
|
|append(...)
| L.append(object) -- append object to end
|
|count(...)
| L.count(value) -> integer -- return number of occurrences of value
|
|extend(...)
| L.extend(iterable) -- extend list by appending elements from the iterable
|
|index(...)
| L.index(value, ]) -> integer -- return first index of value.
| Raises ValueError if the value is not present.
|
|insert(...)
| L.insert(index, object) -- insert object before index
|
|pop(...)
| L.pop() -> item -- remove and return item at index (default last).
| Raises IndexError if list is empty or index is out of range.
|
|remove(...)
| L.remove(value) -- remove first occurrence of value.
| Raises ValueError if the value is not present.
|
|reverse(...)
| L.reverse() -- reverse *IN PLACE*
|
|sort(...)
| L.sort(cmp=None, key=None, reverse=False) -- stable sort *IN PLACE*;
| cmp(x, y) -> -1, 0, 1
|
|----------------------------------------------------------------------
|Data and other attributes inherited from __builtin__.list:
|
|__hash__ = None
|
|__new__ = <built-in method __new__ of type object at 0x822be40>
| T.__new__(S, ...) -> a new object with type S, a subtype of T
FUNCTIONS
adoptNodes(nodes, impl=None)
A special utility method which adopts the given low-level 'nodes' and which
returns a list of high-level equivalents. This is currently experimental and
should not be casually used.
createDocument(namespaceURI, localName, doctype)
createDocumentType(localName, publicId, systemId)
getDOMImplementation()
Return the default DOM implementation.
parse(stream_or_string, html=0, htmlencoding=None, unfinished=0, impl=None)
Parse the given 'stream_or_string', where the supplied object can either be
a stream (such as a file or stream object), or a string (containing the
filename of a document). The optional parameters described below should be
provided as keyword arguments.
If the optional 'html' parameter is set to a true value, the content to be
parsed will be treated as being HTML rather than XML. If the optional
'htmlencoding' is specified, HTML parsing will be performed with the
document encoding assumed to that specified.
If the optional 'unfinished' parameter is set to a true value, unfinished
documents will be parsed, even though such documents may be missing content
such as closing tags.
A document object is returned by this function.
parseFile(filename, html=0, htmlencoding=None, unfinished=0, impl=None)
Parse the file having the given 'filename'. The optional parameters
described below should be provided as keyword arguments.
If the optional 'html' parameter is set to a true value, the content to be
parsed will be treated as being HTML rather than XML. If the optional
'htmlencoding' is specified, HTML parsing will be performed with the
document encoding assumed to that specified.
If the optional 'unfinished' parameter is set to a true value, unfinished
documents will be parsed, even though such documents may be missing content
such as closing tags.
A document object is returned by this function.
parseString(s, html=0, htmlencoding=None, unfinished=0, impl=None)
Parse the content of the given string 's'. The optional parameters described
below should be provided as keyword arguments.
If the optional 'html' parameter is set to a true value, the content to be
parsed will be treated as being HTML rather than XML. If the optional
'htmlencoding' is specified, HTML parsing will be performed with the
document encoding assumed to that specified.
If the optional 'unfinished' parameter is set to a true value, unfinished
documents will be parsed, even though such documents may be missing content
such as closing tags.
A document object is returned by this function.
parseURI(uri, html=0, htmlencoding=None, unfinished=0, impl=None)
Parse the content found at the given 'uri'. The optional parameters
described below should be provided as keyword arguments.
If the optional 'html' parameter is set to a true value, the content to be
parsed will be treated as being HTML rather than XML. If the optional
'htmlencoding' is specified, HTML parsing will be performed with the
document encoding assumed to that specified.
If the optional 'unfinished' parameter is set to a true value, unfinished
documents will be parsed, even though such documents may be missing content
such as closing tags.
XML documents are retrieved using libxml2's own network capabilities; HTML
documents are retrieved using the urllib module provided by Python. To
retrieve either kind of document using Python's own modules for this purpose
(such as urllib), open a stream and pass it to the parse function:
f = urllib.urlopen(uri)
try:
doc = libxml2dom.parse(f, html)
finally:
f.close()
A document object is returned by this function.
toFile(node, filename, encoding=None, prettyprint=0)
Write the serialised form of the given 'node' and its children to a file
having the given 'filename'. The optional 'encoding' can be used to override
the default character encoding used in the serialisation. The optional
'prettyprint' indicates whether the serialised form is prettyprinted or not
(the default setting).
toStream(node, stream, encoding=None, prettyprint=0)
Write the serialised form of the given 'node' and its children to the given
'stream'. The optional 'encoding' can be used to override the default
character encoding used in the serialisation. The optional 'prettyprint'
indicates whether the serialised form is prettyprinted or not (the default
setting).
toString(node, encoding=None, prettyprint=0)
Return a string containing the serialised form of the given 'node' and its
children. The optional 'encoding' can be used to override the default
character encoding used in the serialisation. The optional 'prettyprint'
indicates whether the serialised form is prettyprinted or not (the default
setting).
DATA
HTML_PARSE_NOERROR = 32
HTML_PARSE_NONET = 2048
HTML_PARSE_NOWARNING = 64
XML_NAMESPACE = 'http://www.w3.org/XML/1998/namespace'
XML_PARSE_NOERROR = 32
XML_PARSE_NONET = 2048
XML_PARSE_NOWARNING = 64
__version__ = '0.4.5'
default_impl = <libxml2dom.Implementation object at 0xb748e0ec>
default_ns = {'xml': 'http://www.w3.org/XML/1998/namespace'}
label = 'pi'
null_value_node_types =
value = 7
VERSION
0.4.5
页:
[1]