python简易spider
刚开始学python的时候写的一个python版spider程序相当的简易，不要拍砖
python代码
#-*- encoding:utf-8 -*-
import sgmllib
import urllib2
import threading
import re
import sys
from time import *
SPIDER_ONINIT = 0
SPIDER_ONDATA = 1
SPIDER_ONPARSER = 2
SPIDER_ONFINISH = 3
SPIDER_ONSTOP = 4
#this class is not used now
class SpException(Exception):
def __init__(self,msg):
self.message = msg
def Message(self):
print self.message
class Log:
def __init__(self,filename):
self.log = open(filename,'w')
def WriteLog(self,msg):
self.log.write(msg)
self.log.flush()
#golbal
g_Log = Log(".\\spider_file\\spider.log")
#this class is not used
class ResourcePool:
def __init__(self):
self.res_list = []
self.res_count = 0
self.condition = threading.Condition()
def AddResource(self,data):
self.condition.acquire()
self.res_list.append(data)
self.res_count = self.res_count + 1
self.condition.notify()
self.condition.release()
def GetResource(self):
self.condition.acquire()
while len(self.res_list)0:
self.url = self.url_pool.pop()
if self.IsHandled(self.url) == 0:
g_Log.WriteLog("get "+self.url+" from url_pool\n")
self.filename = self.url
self.ReplaceFileName()
self.status = SPIDER_ONDATA
def OnData(self):
try:
op = urllib2.urlopen(self.url)
f = open(self.dir+self.filename,'w')
f.write(op.read())
except urllib2.URLError:
self.status = SPIDER_ONINIT
except IOError:
self.status = SPIDER_ONINIT
else:
self.url_handled.append(self.url)
g_Log.WriteLog("get "+self.url+"\n")
self.status = SPIDER_ONPARSER
def OnParser(self):
try:
parser = MyHTMLParser(self.url_pool)
f = open(self.dir+self.filename,'r')
parser.feed(f.read())
g_Log.WriteLog("parsed "+self.filename+"\n")
except sgmllib.SGMLParseError:
self.status = SPIDER_ONINIT
except IOError:
self.status = SPIDER_ONINIT
else:
self.status = SPIDER_ONINIT
def OnFinish(self):
if len(self.url_pool) > 0:
self.url = ""
self.filename = ""
self.status = SPIDER_ONINIT
else:
self.status = SPIDER_ONSTOP
def OnStop(self):
self.stop = 1
def ReplaceFileName(self):
error_ch = ['/','\\','*',':','?','"','','|']
for ch in error_ch:
if self.filename.find(ch) >= 0:
self.filename = self.filename.replace(ch,'_')
def IsHandled(self,url):
result = 0
for s in self.url_handled:
if s == url:
result = 1
break
return result
class Spider:
def __init__(self,url):
self.url_pool = []
self.parse_pool = ResourcePool()
self.url_pool.append(url)
def start(self):
self.get_thread = GetHtmlThread(self.url_pool)
self.get_thread.start()
def stop(self):
self.get_thread.stop = 1
self.get_thread.join()
g_Log.WriteLog("get_thread ended")
def usage():
print "*****************************************************"
print "* py_spider vesion 1.0 *"
print "* ctrl-c to stop *"
print "* *"
print "*****************************************************"
if __name__ == '__main__':
try:
usage()
spider = Spider("http://www.pymedia.org")
spider.start()
while 1:
sleep(1)
except KeyboardInterrupt:
print "process is colsing......"
spider.stop()
print "process closed"
页:
[1]