python 日志入库mongodb
今天和大家分享一个python入库mongodb的脚本。。。涉及到python和mongodb，那么安装相应的模块四必不可少的，最简单的安装方法，或者非pip不可了。
1
# pip install pymongo==3.0.4
顺便也记录下源码安装的方式
1
2
3
4
# wget https://pypi.python.org/packages/source/p/pymongo/pymongo-2.8.tar.gz#md5=23100361c9af1904eb2d7722f2658114 --no-check-certificate
# tar xf pymongo-2.8.tar.gz
# cd pymongo-2.8
# python setup.py install
摘自一则日志
1
35783 s100 android 475 192.168.1.100 2015-09-05 08:03:19 strengthenHeroByHeroes{"consume_gold":{"ogold":2893821,"cgold":1700,"gold":2892121,"tag":"strengthenHeroByHeroes"},"taskInfo":[{"id":2310033,"progress":2,"status":0}],"delHeroList":{"id":102014,"id":102014,"id":102014,"id":102010,"id":102010},"id":100026,"olevel":46,"oexp":1700,"cexp":1700,"level":46,"exp":3400} 865982021462182 XiaoMi
入库mongodb的python脚本
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
#!/usr/bin/env python
#coding:utf8
import os
import sys
import json
from pymongo import MongoClient
from datetime import datetime, date, time
def ConMongo(host,port,cur_db,username,password):#链接MongoDB
client = MongoClient(host,port)
db = client
db.authenticate(username,password)
table = db.gamelogs
return table
def parseLog(file_log,Connection):
dic = {}
dl = []
with open(file_log) as fd:
for line in fd:
try:
tokens = line.strip().split('\t')
uid = tokens
server = tokens
system = tokens
level = int(tokens)
vip_level = tokens
ip = tokens
time = datetime.strptime(tokens, "%Y-%m-%d %H:%M:%S") #将时间字符串转换成时间格式
action = tokens
result = json.loads(tokens) #特殊字符串转换成json格式
uuid = tokens
if uid == 'undefined':
if result["game_user_id"]:
uid = result["game_user_id"]
if len(tokens) == 12:
channel = tokens
else:
channel = ''
dic = {'uid':uid,'server':server,'system':system,'level':level,'vip_level':vip_level,'ip':ip,'time':time,'action':action,'result':result,'uuid':uuid,'channel':channel}
dl.append(dic)
if len(dl) == 20000:
Connection.insert_many(dl)
dl = []
except Exception,e:
print e, line
if len(dl) > 0:
Connection.insert_many(dl)
if __name__ == '__main__':
Conn = ConMongo('localhost',27017,'talefundb','talefun','123456')
try:
parseLog(sys.argv,Conn)
except IndexError,e:
print './%s path_logfile' % os.path.basename(__file__)
页:
[1]