Python抓取豆瓣电影详情并提取信息
最近公司有个需求，需要爬取豆瓣上所有电影的名称。写了个脚本，爬取了豆瓣上近10000部电影的详情，包括电影的名称、导演、主演、豆瓣评分、时长、类别、简介等内容。脚本写的简陋，还有一个比较大的问题是如果爬了几次，IP会被豆瓣封掉。后续考虑加入代理来实现。
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
#!/usr/bin/python
# -*-coding:utf-8-*-
# Python: 2.7
# Platform: Mac
# Author: wucl(wucl202000@hotmail.com)
# Program: 爬取豆瓣电影
# Version: 0.1
# History: 2017.8.21
from bs4 import BeautifulSoup
import urllib2, json, random, sys
reload(sys)
sys.setdefaultencoding('utf-8')
def get_data(url):
my_headers = [
'Mozilla/5.0 (Windows NT 5.2) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.122 Safari/534.30',
'Mozilla/5.0 (Windows NT 5.1; rv:5.0) Gecko/20100101 Firefox/5.0',
'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.2; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET4.0E; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET4.0C)',
'Opera/9.80 (Windows NT 5.1; U; zh-cn) Presto/2.9.168 Version/11.50',
'Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/533.21.1 (KHTML, like Gecko) Version/5.0.5 Safari/533.21.1',
'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET4.0E; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET4.0C)']
header = {"User-Agent": random.choice(my_headers)}
req = urllib2.Request(url, headers=header)
html = urllib2.urlopen(req).read()
data = json.loads(html)['data']
return data
def get_movieInfo(url):
my_headers = [
'Mozilla/5.0 (Windows NT 5.2) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.122 Safari/534.30',
'Mozilla/5.0 (Windows NT 5.1; rv:5.0) Gecko/20100101 Firefox/5.0',
'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.2; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET4.0E; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET4.0C)',
'Opera/9.80 (Windows NT 5.1; U; zh-cn) Presto/2.9.168 Version/11.50',
'Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/533.21.1 (KHTML, like Gecko) Version/5.0.5 Safari/533.21.1',
'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET4.0E; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET4.0C)']
header = {"User-Agent": random.choice(my_headers)}
req = urllib2.Request(url, headers=header)
html = urllib2.urlopen(req).read()
soup = BeautifulSoup(html, 'html.parser')
movie = {}
movie['Name'] = soup.find('span',property="v:itemreviewed").text
movie['Year'] = soup.find('span',class_='year').text
movie['Rate'] = soup.find('strong', property="v:average").text
movie['Runtime'] = soup.find('span', property="v:runtime").text
movie['Summary'] = soup.find('span', property='v:summary').text
movie['URL'] = url
movie['Directors'] = ''
directors = soup.find_all('a', rel="v:directedBy")
for director in directors:
movie['Directors'] += director.text
movie['Directors'] += ''
movie['Stars'] = ''
stars = soup.find_all('a', rel="v:starring")
for star in stars:
movie['Stars'] += star.text
movie['Stars'] += ''
movie['Category'] = ''
categorys = soup.find_all('span', property="v:genre")
for category in categorys:
movie['Category'] += category.text
movie['Category'] += ''
return movie
def get_urls():
base_url = 'https://movie.douban.com/j/new_search_subjects?sort=R&range=1,10&tags=%E7%94%B5%E5%BD%B1&start='
urls=[]
nu = 0
while True:
print nu
url = base_url + str(nu)
data = get_data(url)
# print data
if len(data) == 0:
break
for i in data:
urls.append(i['url'])
nu += 20
return urls
if __name__ == '__main__':
urls = get_urls()
f = open('movieinfo.txt','w+')
for url in urls:
try:
movie = get_movieInfo(url)
movie_js = json.dumps(movie)
f.write(movie_js)
f.write('\n')
f.flush()
except:
print url
continue
f.close()
页:
[1]