squid 日志分析
#!/usr/bin/env python#-*-coding:utf8-*-
import os,re,threading,tarfile,shutil,time,sys,datetime
import sys,traceback
sys.path.append('/usr/local/hive/lib/py')
from thrift import Thrift
from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
from hive_service import ThriftHive
from hive_service.ttypes import HiveServerException
transport = TSocket.TSocket('127.0.0.1',10000)
transport = TTransport.TBufferedTransport(transport)
protocol = TBinaryProtocol.TBinaryProtocol(transport)
client = ThriftHive.Client(protocol)
transport.open()
client.execute("use squid")
class threads_del(threading.Thread):
def __init__(self,pt_ip,pt_dt):
threading.Thread.__init__(self)
self.pt_ip = pt_ip
self.pt_dt = pt_dt
def run(self):
client.execute("set hive.exec.dynamic.partition=true")
client.execute("set hive.exec.dynamic.partition.mode=nonstrict")
sql = "alter table squid_log drop partition(pt_ip = '%s', pt_dt=%s)"%(self.pt_ip,self.pt_dt)
client.execute(sql)
def work():
client.execute('show partitions squid_log')
info = client.fetchAll()
infos = {}
for i in info:
a = re.split("/",i)
try:
infos)].append(re.split("=",a))
except:
infos)] = []
infos)].append(re.split("=",a))
for k,v in infos.items():
st = threads_del(k,min(v))
st.start()
st.join()
while threading.active_count() >1:
time.sleep(1)
transport.close()
if __name__ == "__main__":
st = work()
页:
[1]