python 日志入库mongodb

loing

今天和大家分享一个python入库mongodb的脚本。。。

涉及到python和mongodb，那么安装相应的模块四必不可少的，最简单的安装方法，或者非pip不可了。

1	# pip install pymongo==3.0.4

顺便也记录下源码安装的方式

1 2 3 4

# wget https://pypi.python.org/packages ... 904eb2d7722f2658114 --no-check-certificate # tar xf pymongo-2.8.tar.gz # cd pymongo-2.8 # python setup.py install

摘自一则日志

1

35783   s100    android 47  5   192.168.1.100   2015-09-05 08:03:19    strengthenHeroByHeroes  {"consume_gold":{"ogold":2893821,"cgold":1700,"gold":2892121,"tag":"strengthenHeroByHeroes"},"taskInfo":[{"id":2310033,"progress":2,"status":0}],"delHeroList":{"id":102014,"id":102014,"id":102014,"id":102010,"id":102010},"id":100026,"olevel":46,"oexp":1700,"cexp":1700,"level":46,"exp":3400} 865982021462182     XiaoMi

入库mongodb的python脚本

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56

#!/usr/bin/env python #coding:utf8 import os import sys import json from pymongo import MongoClient from datetime import datetime, date, time def ConMongo(host,port,cur_db,username,password):  #链接MongoDB     client = MongoClient(host,port)     db = client[cur_db]     db.authenticate(username,password)     table = db.gamelogs     return table def parseLog(file_log,Connection):     dic = {}     dl = []     with open(file_log) as fd:         for line in fd:         try:             tokens = line.strip().split('\t')             uid = tokens[0]             server = tokens[1]             system = tokens[2]             level = int(tokens[3])             vip_level = tokens[4]             ip = tokens[5]             time = datetime.strptime(tokens[6], "%Y-%m-%d %H:%M:%S")        #将时间字符串转换成时间格式             action = tokens[7]             result = json.loads(tokens[8])                        #特殊字符串转换成json格式             uuid = tokens[9]         if uid == 'undefined':             if result["game_user_id"]:             uid = result["game_user_id"]         if len(tokens) == 12:                 channel = tokens[11]         else:             channel = ''             dic = {'uid':uid,'server':server,'system':system,'level':level,'vip_level':vip_level,'ip':ip,'time':time,'action':action,'result':result,'uuid':uuid,'channel':channel}         dl.append(dic)         if len(dl) == 20000:             Connection.insert_many(dl)             dl = []         except Exception,e:         print e, line     if len(dl) > 0:         Connection.insert_many(dl) if __name__ == '__main__':     Conn = ConMongo('localhost',27017,'talefundb','talefun','123456')     try:         parseLog(sys.argv[1],Conn)     except IndexError,e:     print './%s path_logfile' % os.path.basename(__file__)