Python抓取图片

yw6866

# -*- coding: cp936 -*-
import  urllib
import urllib2
import random
import os,sys
from sgmllib import SGMLParser
class URLLister(SGMLParser):
'''获取html中的图片地址\url地址，装入list中'''
def reset(self):
SGMLParser.reset(self)
self.img = []
self.urls = []
def start_img(self, attrs):
img = [v for k, v in attrs if k=='src']
if img:
self.img.extend(img)
def start_a(self, attrs):
href = [v for k, v in attrs if k=='href']
if href:
self.urls.extend(href)

def get_docum(url):
#url=url+'//'
sock=urllib.urlopen(url)
file=sock.read()
sock.close()
return file
def is_img(url):
global imglenth
request=urllib2.Request(url)
opener=urllib2.build_opener()
try:
con=opener.open(request)
Type=con.headers.dict['content-type'][:5] #判断链接返回的 content-type是不是图片。
Length =int(con.headers.dict['content-length'])#判断图片大小
if Length>imglenth:
return Type
else:
return 0
except:
print sys.exc_info()[0],sys.exc_info()[1]  ##一般来说这样就足够了
print '该图片无法在服务器找到或者图片地址无法识别！'
print url
def get_file_name(ospath,imgname,num):
#name = 'P'+str(random.randint(10000000,99999999))
#filepath = "%s%s.%s" % (ospath,name,(imgname.split('.'))[-1])
#保留原文件名
idx=imgname.rfind("/")
filename=imgname
if idx > -1:
filename=imgname[idx+1:]
filepath=ospath+'%d_'%num+filename
print filepath
return filepath               
def get_img(rq):
parser = URLLister();    doc=get_docum(rq);    parser.feed(doc);    img = parser.img
parser.close()
for i in range(0,len(img)):
if img[0:4]!='http':#处理绝对路径
img=rq+img
return img
def get_url(rq):
parser = URLLister();    doc=get_docum(rq);    parser.feed(doc);    urls = parser.urls
parser.close()
for i in range(0,len(urls)):
if urls[0:4] != 'http': #处理绝对路径
urls = rq+urls
return urls
def depth(url,dep,ospath):
'''三个参数分别是
url ： 需要下载的网站地址
dep ：需要遍历的深度
ospath：图片下载的本地文件夹     
'''
global num
if dep<=0:
return 0
else:
img=get_img(url)
for j in range(0,len(img)):
if is_img(img[j]) == 'image':
filepath = get_file_name(ospath,img[j],num+1);
if (os.path.exists(filepath)):
pass
try:
urllib.urlretrieve(img[j], filepath)
print '已经下载好第%d张图片'%(num+1)
num+=1
except:
print '该图片无法下载或者图片地址无法识别！'
print img[j]
else:
pass
urls=get_url(url)
if len(urls)>0:
for url in urls:
depth(url,dep-1,ospath)
else:
return 0
return 1

if __name__ == '__main__':
imglenth = 1           #设置需要下载的图片大小。
num=0
depth('http://tieba.baidu.com/f?kz=820731958',1,"E:\\python\\GetImageFromWeb\\img\\")   
print '********************************我爬完了！！******************************************'