基于Python的HTTPS协议模拟登陆+爬取页面

gtdfe

之前写的一直没成功，原因是用的不是HTTPS相关的函数。这次仔细研究了一下，有几个需要注意的点，一个是POST模拟登陆的时候，header中的cookie值，不同的网站应该会有不同的要求；另一个是GET页面的时候，是需要加上POST得到的response中的set-cookie的。这样才能利用登陆的成功。    写完POST和GET页面后，顺便写了个简单的命令行实现。

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140

import httplib, urllib import urllib2 import cookielib import sys file_text = "build_change.txt" resultTable = dict() host = 'buuuuuuu.vmware.com' def Login(username, password , csrf =  'Gy2O70iSjOTbWhWgBLvf4HDuf4jUe4RP'): url = '/login/' values = {    'username' : username,    'password' : password,    'next' : '',    'csrfmiddlewaretoken': csrf, }    headers = {      'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.114 Safari/537.36',      'Content-Type': 'application/x-www-form-urlencoded',      'Connection' : 'keep-alive',      'Cookie':'csrftoken=%s' % csrf ,        'Referer':'https://buuuuuuu.vmware.com/login/',      'Origin':'https://buuuuuuu.vmware.com',      'Content-Type':'application/x-www-form-urlencoded',      'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', } values = urllib.urlencode(values) conn = httplib.HTTPSConnection(host, 443) conn.request("POST", url, values, headers) response = conn.getresponse() print 'Login: ', response.status, response.reason ''' hdata = response.getheaders() for i in xrange(len(hdata)):   for j in xrange(len(hdata)):    print hdata[j],   print ''' return response.getheader("set-cookie") def GetHtml(_url , cookie): get_headers = {      'Host' : 'buildweb.eng.vmware.com',      'Connection' : 'keep-alive' ,      'Cache-Control' : 'max-age=0',      'Cookie' : cookie ,      'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',      'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.114 Safari/537.36',      'Accept-Language' : 'zh-CN,zh;q=0.8,en;q=0.6', } conn=httplib.HTTPSConnection(host) conn.request("GET", _url,None,get_headers) res2=conn.getresponse() print "Get %s:" % _url ,res2.status, res2.reason ''' hdata1 = res2.getheaders() for i in xrange(len(hdata1)):   for j in xrange(len(hdata1)):    print hdata1[j],   print ''' data = res2.read() fp = open("build_change.txt","w") fp.write(data) fp.close() def ParseHtml(): fp = open(file_text,"r") content = fp.readline() _pos = 0 while content:     if content.find("class=\"change-body\"") >= 0:    topic = content.split(">")    resultTable[_pos] = topic[1]    while content:     content = fp.readline()     resultTable[_pos] = resultTable[_pos] + content     if content.find("</div>")>= 0:      _pos = _pos + 1      break   content = fp.readline() fp.close() print "Parse html success." def GenerateResultTxt(): f = open("build_change_result.txt","w") for m in resultTable.keys():   f.write("-------------------------------------------------------------------------------------------\n")   f.write(resultTable[m]) f.close() print "Generate result success : build_change_result.txt ." def Help(): print '-h    :    help' print '-u    :    username(must)' print '-p    :    password(must)' print '-c    :    csrftoken(optional)' print '-s    :    sandbox build id(must)' print 'For example:' print '[1]  python BuildChange.py -h' print '[2]  python BuildChang.py -u u -p p -s s1 s2' print '[3]  python BuildChang.py -u u -p p -c c -s s1 s2' def ParseParam(com): length = len(com) username = "" password = "" csrf = "" sid1 = "" sid2 = "" if length == 2 or length == 8 or length == 10:   if com[1] == '-h':    Help()   for i in range(1,length):    if com == '-u' and i < (length-1):     username = com[i+1]     i += 1    elif com == '-p' and i < (length-1):     password = com[i+1]     i += 1    elif com == '-c' and i < (length-1):     csrf = com[i+1]     i += 1    elif com == '-s' and i < (length-2):     sid1 = com[i+1]     sid2 = com[i+2]     i += 2 if username == "" or password == "" or sid1 == "" or sid2 == "":   print '[Error] Parameter error!'   print '[Error] You can use \"python BuildChange.py -h\" to see how can use this script. ' else:   if csrf == "":    cookie = Login(username, password)   else:    cookie = Login(username, password, csrf)   _url = "//changelog//between//%s//and//%s/" % (sid1, sid2)   GetHtml(_url, cookie)   ParseHtml()   GenerateResultTxt() # C:\Python27\python.exe C:\Users\knight\Desktop\build\BuildChange.py -u xux -p KKKKKKKK -s 1859409 1858525 if __name__ == "__main__": ParseParam(sys.argv)