解决线上故障-python分析日志脚本

FXMAR

　　简直在作死的节奏，一个项目出了三个故障，半夜被搞醒处理故障。。。
　　末了，需要从日志里面扣出数据，进行历史数据修复
　　开始用shell,奈何shell的水准半桶水都不到。。实在不行，果断用python.磕磕碰碰，一路百度。。终于完成了。
　　贴出来。。半夜的成果

(1)对于gz文件，已上面其中一个文件为例，执行脚本sudo python parse_gz_log.py  /home/q/www/hms.***.com/logs/access.2014-01-09.log.gz会在/home/q/www/hms.***.com/logs/下面生成相应的sql文件access.2014-01-09.log.gz.sql。



 

（2）对于唯一的log文件access.2014-01-10.log，执行如下脚本sudo python parselog.py  /home/q/www/***.***.com/logs/access.2014-01-10.log会在/home/q/www/***.***.com/logs/下面生成相应的sql文件access.2014-01-10.log.sql
　　代码：

# -*- coding: UTF-8 -*-
import sys
#line="192.168.224.18 - - [10/Jan/2014:02:05:42 +0800] \"GET /commission/order/user/edit.htm?fromDate=2014-01-12&toDate=2014-01-13&roomId=1147845&QHFP=ZSD_A9F061AB&QHP=ZSB_A9F08213&from=q***rHotel&partnerUserId=&bd_sign= HTTP/1.0\" 200 39745 \"http://hotel.q***r.com/booksystem/Booking_Main.jsp?local=zh&full=false&ttsSign=05facbd629f70c21d00c1e7353a7060a&priceCut=0&tid=5357743&required1=2014-01-12&required2=2014-01-13&payment=0&CPCB=wiqu***ar0004&roomId=1147845&requestID=c0a8f8af-m2ijp-6mn&lpsp=np&ppb=0&stat=30148&retailPrice=1&detailType=guru&from=q***rHotel&sgroup=A&filterid=973515f5-5323-4abf-bd47-68804ba091b6_C&QHFP=ZSD_A9F061AB&QHP=ZSB_A9F08213&stat2=1124511&required0=concepcion&codeBase=wiqun***r0004&hotelSEQ=concepcion_44" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36" "U.pudh0348" "192.168.0.207_429c0d20_13a91dbf8d5_66ea|1351068665199\" 192.168.11.202"

indexstr="commission/order/user/edit.htm"
indexfrom="fromDate="
indextodate="&toDate="
indexroomId="&roomId="
indexroomIdEnd="&QHFP="
data={}
#fileName="access.2014-01-10.log"
#需要分析的日志文件
fileName= sys.argv[1]
orderdate=fileName.split('.')[1]
#生成sql语句
#sqlFile="update_username.sql"
sqlFile=fileName+".sql"
sf=open(sqlFile,'w')
readfile=open(fileName)
for line in readfile:
#判断是否包含commission/order/user/edit.htm
if(line.find(indexstr)<0 or line.find(indexfrom)<0 or line.find(indextodate)<0 or line.find(indexroomId)<0):
continue  
fromdate=line[line.find(indexfrom) + 9:line.find(indextodate)]
todate=line[line.find(indextodate) + 8:line.find(indexroomIdEnd)]
todate=todate[0:todate.find(indexroomId)]   
roomId=line[line.find(indextodate) + 8:line.find(indexroomIdEnd)]
roomId=roomId[roomId.find(indexroomId)+8:]

#获取username
list=line.split(' ')
if(len(list)<3):continue
username=list[len(list)-3]
username=username[username.find("U.") + 2:username.find("192.168.")]
if(username == '-' or username ==''):continue
userdate=fromdate+","+todate+","+roomId+","+username
#去除重复数据
data[userdate]="1"
print fromdate,todate,roomId,username
#循环数据，写入文本文件
for key in data.keys():
keys=key.split(',')
if(len(keys)<3):continue
sql="update commission_order set user_name='"+keys[3]+"' where from_date='"+keys[0]+"' and to_date='"+keys[1]+"' and room_id='"+keys[2]+"' and order_date='"+orderdate+"';"
sf.write(sql+"\n")
sf.close()
readfile.close()

　　下面是处理gz文件的代码：

# -*- coding: UTF-8 -*-
import sys
import gzip
#line="192.168.224.18 - - [10/Jan/2014:02:05:42 +0800] \"GET /commission/order/user/edit.htm?fromDate=2014-01-12&toDate=2014-01-13&roomId=1147845&QHFP=ZSD_A9F061AB&QHP=ZSB_A9F08213&from=***&partnerUserId=&bd_sign= HTTP/1.0\" 200 39745 \"http://hotel.***.com/booksystem/Booking_Main.jsp?local=zh&full=false&ttsSign=05facbd629f70c21d00c1e7353a7060a&priceCut=0&tid=5357743&required1=2014-01-12&required2=2014-01-13&payment=0&CPCB=wiqu***ar0004&roomId=1147845&requestID=c0a8f8af-m2ijp-6mn&lpsp=np&ppb=0&stat=30148&retailPrice=1&detailType=guru&from=***&sgroup=A&filterid=973515f5-5323-4abf-bd47-68804ba091b6_C&QHFP=ZSD_A9F061AB&QHP=ZSB_A9F08213&stat2=1124511&required0=concepcion&codeBase=***&hotelSEQ=concepcion_44" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36" "U.pudh0348" "192.168.0.207_429c0d20_13a91dbf8d5_66ea|1351068665199\" 192.168.11.202"

indexstr="/commission/order/user/edit.htm"
indexfrom="fromDate="
indextodate="&toDate="
indexroomId="&roomId="
indexroomIdEnd="&QHFP="
data={}
#fileName="access.2014-01-09.log.gz"
#需要分析的日志文件
fileName= sys.argv[1]
orderdate=fileName.split('.')[1]
#生成sql语句
#sqlFile="update_username.sql"
sqlFile=fileName+".sql"
sf=open(sqlFile,'w')
readfile = gzip.GzipFile(fileName)
for line in readfile:
#判断是否包含commission/order/user/edit.htm
if(line.find(indexstr)<0 or line.find(indexfrom)<0 or line.find(indextodate)<0 or line.find(indexroomId)<0):
continue  
fromdate=line[line.find(indexfrom) + 9:line.find(indextodate)]
todate=line[line.find(indextodate) + 8:line.find(indexroomIdEnd)]
todate=todate[0:todate.find(indexroomId)]   
roomId=line[line.find(indextodate) + 8:line.find(indexroomIdEnd)]
roomId=roomId[roomId.find(indexroomId)+8:]

#获取username
list=line.split(' ')
if(len(list)<3):continue
username=list[len(list)-3]
username=username[username.find("U.") + 2:username.find("192.168.")]
if(username == '-' or username ==''):continue
userdate=fromdate+","+todate+","+roomId+","+username
#去除重复数据
data[userdate]="1"
print fromdate,todate,roomId,username
#循环数据，写入文本文件
for key in data.keys():
keys=key.split(',')
if(len(keys)<3):continue
sql="update commission_order set user_name='"+keys[3]+"' where from_date='"+keys[0]+"' and to_date='"+keys[1]+"' and room_id='"+keys[2]+"' and order_date='"+orderdate+"';"
sf.write(sql+"\n")
sf.close()
readfile.close()

　　自己mark一下。。。