python爬取邮箱的实例代码
不用编码也是可以匹配邮箱的。下面是一个用队列实现，广度优先的简单爬虫代码。先就这样吧，目测暂时不会再理它了，以后有时间再修改。
1 # -*- coding: cp936 -*-
2import urllib2
3import re
4from pyquery import PyQuery as pq
5from lxml import etree
6
7#mailpattern = re.compile('[^\._:>\\-][\w\.-]+@(?:+\.)++')
8mailpattern = re.compile('+@(?:+\.)++')
9
10htmlcount = 0#to count the urls
11maxcount = 3000 # the max count
12allUrls = set()
13allMails = set()
14UrlsQlist = []
15UrlsQdict = {}
16url = "http://www.163.com"
17fmails = open("E:/py/crawler/mailresult.txt","a")
18furls = open("E:/py/crawler/urlresult.txt","a")
19
20
21
22
23def geturls(data):#the function to get the urls in the html
24 urls = set()
25 if data:
26 d = pq(data)
27 label_a = d.find('a')#用pyquery库去找到 a 标签.
28 if label_a:
29 label_a_href = d('a').map(lambda i,e:pq(e)('a').attr('href'))
30 for u in label_a_href:
31 if u!="javascript" :
32 if u == "http":
33 urls.add(u)
34 else:
35 urls.add(url + u)
36 #for u in urls:
37 #print u
38 return urls
39 else:
40 return None
41
42def gethtml(url):
43 try:
44 fp = urllib2.urlopen(url)
45 except:
46 print "urllib2.urlopen error"
47 return None
48 else:
49 mybytes =fp.read()
50 fp.close()
51 return mybytes
52
53def savemails(data): # the function to save the emails
54 if data:
55 mailResult = mailpattern.findall(data)
56 mailResultset = set(mailResult)
57 if mailResultset:
58 allMails.update(mailResultset)
59
60def savehtml(pagecontent,count):
61 if pagecontent != None:
62 f = open("E:/py/crawler/html/"+str(count)+".html","w")
63 f.write(pagecontent)
64 f.close()
65 else:
66 f = open("E:/py/crawler/html/"+str(count)+"error"+".html","w")
67 f.write("this page empty")
68 f.close()
69
70def BFS(firstUrl):
71 global htmlcount
72 global maxcount
73 allUrls.add(firstUrl)
74 UrlsQlist = list(allUrls)
75 while htmlcount < maxcount : #数量小于最大值
76 tempUrl = UrlsQlist.pop(0)# the queue
77 myWebStr = gethtml(tempUrl)
78 savehtml(myWebStr,htmlcount)
79 savemails(myWebStr)
80 firstUrls_set = geturls(myWebStr)#初始页面的处理
81 if firstUrls_set != None:
82 allUrls.update(firstUrls_set) #记录全部 url
83 for u in firstUrls_set:
84 if u not in UrlsQlist:
85 UrlsQlist.append(u)
86 htmlcount = htmlcount + 1
87
88
89BFS(url)
90for u in allMails:
91 try:
92 fmails.write(u)
93 fmails.write('\n')
94 except:
95 continue
96for u in allUrls:
97 try:
98 furls.write(u)
99 furls.write('\n')
100 except:
101 continue
102fmails.close()
103furls.close()
2013.5.13 update
本来想在加个多线程。。。。结果看了 好多资料 无处下手,再研究研究 ，日后再改
加了点 url规范化。代码整理如下：
1 import urllib2
2 import re
3 from pyquery import PyQuery as pq
4 from lxml import etree
5 import urlparse
6 import time
7
8 allUrls = set()
9 allMails = set()
10 urlsDownlist = []
11
12 class mailCrawler:
13 def __init__(self,mailExpression,start_url,maxcount):
14 ''' mailExpressoin 邮箱的正则表达式;
15 start_url开始邮箱；
16 maxcount最大数量'''
17 self.mailpattern = re.compile(mailExpression)
18 self.maxcount = maxcount
19 self.htmlcount = 0
20 self.UrlsQlist = []#url queue 实现广度优先
21 self.url = start_url
22
23
24 def url_normal(self,url):
25 '''url 规范化 '''
26 scheme,netloc,path,query = urlparse.urlsplit(url)[:4]
27 netloc = netloc.lower()
28
29 url.encode("utf-8")
30
31 if path:
32 path = re.sub('/{2,}','/',path)#去除url中的重复/
33 path = re.sub(r'\.$','',path)#去除url中结尾多余的点
34 path = re.sub('/$','',path)#去除url中结尾多余的/
35 path = re.sub('\s','',path)#取出url中的空格
36 if query:
37 return '%s://%s%s?%s' % (scheme,netloc,path or '/',query)
38 else:
39 return '%s://%s%s' % (scheme,netloc,path)
40
41 def geturls(self,data):
42 '''解析html中的url'''
43 urls = set()
44 if data:
45 d = pq(data)
46 label_a = d.find('a')#用pyquery库去找到 a 标签.
47 if label_a:
48 label_a_href = d('a').map(lambda i,e:pq(e)('a').attr('href'))
49 for u in label_a_href:
50 if u!="javascript" and u!="mailto" :
51 if u == "http":
52 normal_url = self.url_normal(u)
53 urls.add(normal_url)
54 else:
55 normal_url = self.url_normal(self.url + u)
56 urls.add(normal_url)
57 return urls
58 else:
59 return None
60
61 def gethtml(self,url):
62 '''下载html5s超时'''
63 try:
64 fp = urllib2.urlopen(url,None,5)
65 except:
66 print "urllib2.urlopen erroror timeout"
67 return None
68 else:
69 mybytes =fp.read()
70 fp.close()
71 return mybytes
72
73 def savemails(self,data):
74 '''将抓取到的url存放到 allmails中 ，set去重复'''
75 global allMails
76 if data:
77 mailResult = self.mailpattern.findall(data)
78 mailResultset = set(mailResult)
79 if mailResultset:
80 allMails.update(mailResultset)
81
82 def savehtml(self,pagecontent,htmlcount,url):
83 '''保存html文件 '''
84 if pagecontent != None:
85 f = open("E:/py/crawler/html/"+str(htmlcount)+".html","w")
86 f.write(pagecontent)
87 f.close()
88 else:
89 f = open("E:/py/crawler/html/"+str(htmlcount)+"error"+".html","w")
90 try:
91 f.write(url)
92 except:
93 f.write("encode error")
94 f.close()
95
96 def BFS(self):
97 '''用队列实现广度优先，爬取url '''
98 global allUrls
99 global urlsDownlist
100 allUrls.add(self.url)
101 self.UrlsQlist = list(allUrls)
102 while self.htmlcount < self.maxcount : #数量小于最大值
103 tempUrl = self.UrlsQlist.pop(0)# the queue
104 print tempUrl
105 urlsDownlist.append(tempUrl)
106 myWebStr = self.gethtml(tempUrl)
107 self.savehtml(myWebStr,self.htmlcount,tempUrl)
108 self.savemails(myWebStr)
109 firstUrls_set = self.geturls(myWebStr)#初始页面的处理
110 if firstUrls_set != None:
111 for u in firstUrls_set:
112 if u not in allUrls:
113 allUrls.add(u)
114 self.UrlsQlist.append(u)
115 self.htmlcount = self.htmlcount + 1
116
117
118 def main():
119 reg = r'+@(?:+\.)++'
120 url = "http://www.baidu.com"
121 count = 100
122 fmails = open("E:/py/crawler/mailresult.txt","a")
123 furls = open("E:/py/crawler/urlresult.txt","a")
124 fdownUrls = open("E:/py/crawler/urlDownresult.txt","a")
125 newcrawler = mailCrawler(reg,url,count)
126 newcrawler.BFS()
127 for u in allMails:
128 try:
129 fmails.write(u)
130 fmails.write('\n')
131 except:
132 continue
133 for u in allUrls:
134 try:
135 furls.write(u)
136 furls.write('\n')
137 except:
138 continue
139 for u in urlsDownlist:
140 try:
141 fdownUrls.write(u)
142 fdownUrls.write('\n')
143 except:
144 continue
145 fmails.close()
146 furls.close()
147 fdownUrls.close()
148
149 if __name__ == '__main__':
150 main()
页:
[1]