squid 日志分析

eddik

　　#!/usr/bin/env python
#-*-coding:utf8-*-

import os,re,threading,tarfile,shutil,time,sys,datetime
import sys,traceback
sys.path.append('/usr/local/hive/lib/py')
from thrift import Thrift
from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
from hive_service import ThriftHive
from hive_service.ttypes import HiveServerException

transport = TSocket.TSocket('127.0.0.1',10000)
transport = TTransport.TBufferedTransport(transport)
protocol = TBinaryProtocol.TBinaryProtocol(transport)
client = ThriftHive.Client(protocol)
transport.open()
client.execute("use squid")


class threads_del(threading.Thread):
    def __init__(self,pt_ip,pt_dt):
        threading.Thread.__init__(self)
        self.pt_ip = pt_ip
        self.pt_dt = pt_dt

    def run(self):
        client.execute("set hive.exec.dynamic.partition=true")
        client.execute("set hive.exec.dynamic.partition.mode=nonstrict")
        sql = "alter table squid_log drop partition(pt_ip = '%s', pt_dt=%s)"%(self.pt_ip,self.pt_dt)
        client.execute(sql)

def work():
    client.execute('show partitions squid_log')
    info = client.fetchAll()
    infos = {}
    for i in info:
        a = re.split("/",i)
        try:
            infos[re.split("=",a[0])[1]].append(re.split("=",a[1])[1])
        except:
            infos[re.split("=",a[0])[1]] = []
            infos[re.split("=",a[0])[1]].append(re.split("=",a[1])[1])

    for k,v in infos.items():
        st = threads_del(k,min(v))
        st.start()
        st.join()

    while threading.active_count() >1:
        time.sleep(1)

    transport.close()


if __name__ == "__main__":
    st = work()