【Python真的很强大】使用casperjs获取javascript渲染生成的html内容

sakko51150

 文章摘要：其实这里casperjs与python没有直接关系,主要依赖casperjs调用phantomjs webkit获取html文件内容。长期以来，爬虫抓取 客户端javascript渲染生成的html页面 都极为 困难, Java里面有 HtmlUnit, 而Python里，我们可以使用独立的跨平台的CasperJS。
 

• 创建site.js(接口文件，输入:url，输出:html file)
  //USAGE: E:\toolkit\n1k0-casperjs-e3a77d0\bin>python casperjs site.js --url=http://spys.ru/free-proxy-list/IE/ --outputfile='temp.html'
  var fs = require('fs');
  var casper = require('casper').create({
  pageSettings: {
  loadImages: false,      
  loadPlugins: false,      
  userAgent: 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36 LBBROWSER'
  },
  logLevel: "debug",//日志等级
  verbose: true    // 记录日志到控制台
  });
  var url = casper.cli.raw.get('url');
  var outputfile = casper.cli.raw.get('outputfile');
  //请求页面
  casper.start(url, function () {
  fs.write(outputfile, this.getHTML(), 'w');
  });
  casper.run();
  
• python 代码, checkout_proxy.py
  import json
  import sys
  #import requests
  #import requests.utils, pickle
  from bs4 import BeautifulSoup
  import os.path,os
  import threading
  #from multiprocessing import Process, Manager
  from datetime import datetime
  import traceback
  import logging
  import re,random
  import subprocess
  import shutil
  import platform
  
  
  output_file =  os.path.join(os.path.dirname(os.path.realpath(__file__)),'proxy.txt')
  global_log  = 'http_proxy' + datetime.now().strftime('%Y-%m-%d') + '.log'
  if not os.path.exists(os.path.join(os.path.dirname(os.path.realpath(__file__)),'logs')):
  os.mkdir(os.path.join(os.path.dirname(os.path.realpath(__file__)),'logs'))
  global_log = os.path.join(os.path.dirname(os.path.realpath(__file__)),'logs',global_log)
  logging.basicConfig(level=logging.DEBUG,format='[%(asctime)s] [%(levelname)s] [%(module)s] [%(funcName)s] [%(lineno)d] %(message)s',filename=global_log,filemode='a')
  log = logging.getLogger(__name__)
  #manager = Manager()
  #PROXY_LIST = manager.list()
  mutex = threading.Lock()
  PROXY_LIST = []
  
  def isWindows():
  if "Windows" in str(platform.uname()):
  return True
  else:
  return False
  
  def getTagsByAttrs(tagName,pageContent,attrName,attrRegValue):
  soup = BeautifulSoup(pageContent)                                                                                               
  return soup.find_all(tagName, { attrName : re.compile(attrRegValue) })
  
  def getTagsByAttrsExt(tagName,filename,attrName,attrRegValue):
  if os.path.isfile(filename):
  f = open(filename,'r')      
  soup = BeautifulSoup(f)
  f.close()
  return soup.find_all(tagName, { attrName : re.compile(attrRegValue) })
  else:
  return None
  
  class Site1Thread(threading.Thread):
  def __init__(self,outputFilePath):
  threading.Thread.__init__(self)
  self.outputFilePath = outputFilePath
  self.fileName = str(random.randint(100,1000)) + ".html"
  self.setName('Site1Thread')
  def run(self):
  site1_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),'site.js')
  site2_file = os.path.join(self.outputFilePath,'site.js')
  if not os.path.isfile(site2_file) and os.path.isfile(site1_file):
  shutil.copy(site1_file,site2_file)
  #proc = subprocess.Popen(["bash","-c", "cd %s && ./casperjs site.js --url=http://spys.ru/free-proxy-list/IE/ --outputfile=%s" % (self.outputFilePath,self.fileName) ],stdout=subprocess.PIPE)
  if isWindows():
  proc = subprocess.Popen(["cmd","/c", "%s/casperjs site.js --url=http://spys.ru/free-proxy-list/IE/ --outputfile=%s" % (self.outputFilePath,self.fileName) ],stdout=subprocess.PIPE)
  else:
  proc = subprocess.Popen(["bash","-c", "cd %s && ./casperjs site.js --url=http://spys.ru/free-proxy-list/IE/ --outputfile=%s" % (self.outputFilePath,self.fileName) ],stdout=subprocess.PIPE)
  out=proc.communicate()[0]
  htmlFileName = ''
  #因为输出路径在windows不确定，所以这里加了所有可能的路径判断
  if os.path.isfile(self.fileName):
  htmlFileName = self.fileName
  elif os.path.isfile(os.path.join(self.outputFilePath,self.fileName)):
  htmlFileName = os.path.join(self.outputFilePath,self.fileName)
  elif os.path.isfile(os.path.join(os.path.dirname(os.path.realpath(__file__)),self.fileName)):
  htmlFileName = os.path.join(os.path.dirname(os.path.realpath(__file__)),self.fileName)
  if (not os.path.isfile(htmlFileName)):
  print 'Failed to get html content from http://spys.ru/free-proxy-list/IE/'
  print out
  sys.exit(3)
  mutex.acquire()
  PROXYList= getTagsByAttrsExt('font',htmlFileName,'class','spy14$')
  for proxy in PROXYList:
  tdContent = proxy.renderContents()
  lineElems = re.split('[<>]',tdContent)
  if re.compile(r'\d+').search(lineElems[-1]) and re.compile('(\d+\.\d+\.\d+)').search(lineElems[0]):
  print lineElems[0],lineElems[-1]
  PROXY_LIST.append("%s:%s" % (lineElems[0],lineElems[-1]))
  mutex.release()
  try:
  if os.path.isfile(htmlFileName):
  os.remove(htmlFileName)
  except:
  pass
  if __name__ == '__main__':
  try:
  if(len(sys.argv)) < 2:
  print "Usage:%s [casperjs path]" % (sys.argv[0])
  sys.exit(1)
  if not os.path.exists(sys.argv[1]):
  print "casperjs path: %s does not exist!" % (sys.argv[1])
  sys.exit(2)
  if os.path.isfile(output_file):
  f = open(output_file)
  lines = f.readlines()
  f.close
  for line in lines:
  PROXY_LIST.append(line.strip())
  thread1 = Site1Thread(sys.argv[1])
  thread1.start()
  thread1.join()
  f = open(output_file,'w')
  for proxy in set(PROXY_LIST):
  f.write(proxy+"\n")
  f.close()
  print "Done!"
  except SystemExit:
  pass
  except:
  errMsg = traceback.format_exc()
  print errMsg
  log.error(errMsg)