Python superbwallpapers 动漫分类 下载

gfdxy3322

　　动漫分类壁纸多线程下载,有Bug



# -*- coding: utf-8 -*-
import os,urllib2,re,urllib
from bs4 import BeautifulSoup
import socket
socket.setdefaulttimeout(25)      #in case of overtime:http://outofmemory.cn/code-snippet/16848/python-through-urllib-urlretrieve-file-setting-method
#another way:urllib.request.socket.setdefaulttimeout(20)
re_link = re.compile(r'/anime/.{0,50}-\d{5}')
re_404 = re.compile(r'Page not found - Please try some of the popular items below')  
main_url = []
#main_url.append("http://www.superbwallpapers.com/anime/")
pic_page = []
pic_name = []
pic_url = []
pic_url_number =[]
end_page = 40
for each_page in range(end_page):
main_url.append("http://www.superbwallpapers.com/anime/" + str(each_page + 1) + ".html")
each_page += 1
print main_url
#how_many = 0        
def one_page(main_url):
#global how_many
main_page_html = urllib2.urlopen(main_url).read()
soup = BeautifulSoup(main_page_html,fromEncoding="gb18030")
match_pic = []
for link in soup.find_all('a'):
href = str(link.get('href'))   
match = re_link.match(href)
if match :
#print match.group()
            match_pic.append(match.group())
global pic_url_number
global pic_page
global pic_name
global pic_url
for i in range((len(match_pic)-1)/2):
pic_page.append('http://www.superbwallpapers.com' + match_pic[i*2])
pic_name.append(match_pic[i*2][7:])
pic_url.append("http://cdn.superbwallpapers.com/wallpapers" + match_pic[i*2] + "-1920x1080.jpg")
pic_url_number.append(x)
output = open('K://PIC/url.txt','w+')
for x in range(end_page):
one_page(main_url[x])
title = "K://PIC/" + str(x)
if not os.path.isdir(title):
os.mkdir(title)
output.write(str(pic_url))
output.close()
pic_number = 0
url_fail = []
import threading
how_many = 0
lock = threading.Lock()
#one_page(main_url[how_many])
class myThread (threading.Thread):
def __init__(self, pic_url):
threading.Thread.__init__(self)
self.pic_url = pic_url
def run(self):
global pic_number
global how_many
print '%s acquire lock...' % threading.currentThread().getName()
lock.acquire()
print '%s get the lock.' % threading.currentThread().getName()
o1 = pic_number
pic_number += 1
# 释放锁
        lock.release()
print '%s release lock...' % threading.currentThread().getName()
try:
urllib.urlretrieve(pic_url[o1], "K://PIC/" + str(pic_url_number[o1])+ "/" + str(pic_name[o1]) + ".jpg")   
#detail:http://www.nowamagic.net/academy/detail/1302861
except:  #except socket.timeout as e:
try:
urllib.urlretrieve(pic_url[o1], "K://PIC/" + str(pic_url_number[o1])+ "/" + str(pic_name[o1]) + ".jpg")
except:  
global url_fail
url_fail.append(pic_url[o1])
url_fail.append(pic_url_number[o1])
print "-----socket timout-----,record..."
print "Picture " + str(pic_name[o1]) + " Downloaded"   
def start_new_thread():
thread = myThread(pic_url)
thread.start()
while pic_number <= len(pic_url):
if threading.activeCount() < 7:
start_new_thread()