python scrapy+Mongodb爬取蜻蜓FM，酷我及懒人听书

styxmx

def get_qt_jmlist(self, response):　　jm_json = json.loads(response.body, encoding="utf-8")
　　de = response.meta["de"]
　　for jm_data in jm_json["data"]:
　　if jm_data is None:
　　pass
　　else:
　　try:
　　file_path = "http://upod.qingting.fm/%(file_path)s?deviceid=ffffffff-ebbe-fdec-ffff-ffffb1c8b222" % \
　　jm_data["mediainfo"]["bitrates_url"][0]
　　item = QtscrapyItem()
　　# print(item)
　　# print(jm_data["id"])
　　item["id"] = str(jm_data["id"])
　　parent_info = "%(parent_id)s_%(parent_name)s" % de["parent_info"]
　　item["parent_info"] = parent_info
　　item["title"] = jm_data["title"]
　　item["update_time"] = str(jm_data["update_time"])[:str(jm_data["update_time"]).index(' ')]
　　item["file_path"] = file_path
　　item["source"] = "qingting"
　　yield item
　　except:
　　pass
　　pass
　　def get_kw_catlist(self, response):
　　try:
　　kw_json = json.loads(response.body, encoding="utf-8")
　　if kw_json["sign"] is not None:
　　if kw_json["list"] is not None:
　　for data in kw_json["list"]:
　　p_id = data["Id"]
　　kw_p_url = "http://ts.kuwo.cn/service/getlist.v31.php?act=cat&id=%s&type=hot" % p_id
　　yield Request(kw_p_url, callback=self.get_kw_cat)
　　except:
　　print("*" * 300)
　　print(self.name, kw_json)
　　pass
　　def get_kw_cat(self, response):
　　try:
　　kw_json = json.loads(response.body, encoding="utf-8")
　　p_info = {}
　　if kw_json["sign"] is not None:
　　if kw_json["list"] is not None:
　　for data in kw_json["list"]:
　　id = data["Id"]
　　p_info["p_id"] = data["Id"]
　　p_info["p_name"] = data["Name"]

　　kw_pp_url = "http://ts.kuwo.cn/service/getlist.v31.php?act=detail&id=%s" %>　　yield Request(kw_pp_url, callback=self.get_kw_jmlist, meta={"p_info": p_info})
　　except:
　　print("*" * 300)
　　print(self.name, kw_json)
　　pass
　　def get_kw_jmlist(self, response):
　　jm_json = json.loads(response.body, encoding="utf-8")
　　p_info = response.meta["p_info"]
　　for jm_data in jm_json["Chapters"]:
　　if jm_data is None:
　　pass
　　else:
　　try:
　　file_path = "http://cxcnd.kuwo.cn/tingshu/res/WkdEWF5XS1BB/%s" % jm_data["Path"]
　　item = QtscrapyItem()
　　item["id"] = str(jm_data["Id"])
　　parent_info = "%(p_id)s_%(p_name)s" % p_info
　　item["parent_info"] = parent_info
　　item["title"] = jm_data["Name"]
　　item["update_time"] = ""
　　item["file_path"] = file_path
　　item["source"] = "kuwo"
　　yield item
　　except:
　　pass
　　pass
　　def get_lr_booklist(self, response):
　　s_lr_json = json.loads(response.body, encoding="utf-8")
　　if len(s_lr_json["list"]) > 0:
　　for s_lr in s_lr_json["list"]:
　　s_lr_url = "http://api.mting.info/yyting/bookclient/ClientGetBookResource.action?bookId=%(id)s&pageNum=1&pageSize=2000&sortType=0&token=_4WfzpCah8ujgJZZzboaUGkJQvWGfEEL-zdukwv7lbY*&imei=ODY1MTY2MDIxNzMzNjI0" % s_lr
　　meta = {}
　　meta["id"] = s_lr["id"]
　　meta["name"] = s_lr["name"]
　　yield Request(s_lr_url, callback=self.get_lr_kmlist, meta={"meta": meta})
　　def get_lr_kmlist(self, response):
　　ss_lr_json = json.loads(response.body, encoding="utf-8")
　　parent = response.meta["meta"]
　　if len(ss_lr_json["list"]) > 0:
　　for ss_lr in ss_lr_json["list"]:
　　try:
　　item = QtscrapyItem()
　　item["id"] = str(ss_lr["id"])
　　parent_info = "%(id)s_%(name)s" % parent
　　item["parent_info"] = parent_info
　　item["title"] = ss_lr["name"]
　　item["update_time"] = ""
　　item["file_path"] = ss_lr["path"]
　　item["source"] = "lr"
　　yield item
　　except:
　　pass