python 增量同步远程文件
因为发的时候，有些敏感信息没去除，所以大家发文的时候，切记要过滤。不然抓取的很厉害。这个脚本的作用是同步远程机器的少量log到一台机器上，然后通过logstash过滤显示犯了好几个错误[*]local写错locals，locals是python特有定义变量的，不能重名
[*]sftp老报错，IOError: No such file,这个是因为没有创建本地目录
[*]第一次全量同步，第二次同步五分钟之内修改的文件
[*]特别对列表的处理，用strip去除\n
[*]read()和readlines的区别，其实python基础编程类书籍有讲到
演示
cat filelistx.x.x.x::password::tomcat::/home/python/::/opt/logs/::
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
#!/usr/bin/python
import pexpect
import paramiko
import os
import sys
import time
import multiprocessing
import datetime
import crash_on_ipy
from stat import S_ISDIR
ip_list = []
#room_id = sys.argv
class run_cmd():
def __init__(self,hostname=None,password=None,username=None,port=None,echo_cmd=None):
#threading.Thread.__init__(self)
self.hostname=hostname
self.password=password
self.username=username
self.port=port
self.echo_cmd=echo_cmd
#self.thread_stop=False
def run(self):
paramiko.util.log_to_file('paramiko.log')
s=paramiko.SSHClient()
s.set_missing_host_key_policy(paramiko.AutoAddPolicy())
s.connect(hostname = self.hostname,username=self.username, password=self.password)
stdin,stdout,stderr=s.exec_command(self.echo_cmd)
return stdout.readlines()
s.close()
def stop(self):
self.thread_stop=True
class get_thread():
def __init__(self,hostname,password,username,port=None):
#def __init__(self,hostname,username='root',key_file=None,password=None):
#def __init__(self,hostname=None,password=None,username=None,port=None,local_dir=None,remote_dir=None):
self.hostname = hostname
self.username = username
self.password = password
self.scp = paramiko.SSHClient()
self.scp.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.scp = paramiko.Transport((hostname, 22))
self.scp.connect(username=username, password=password)
self.sftp = paramiko.SFTPClient.from_transport(self.scp)
def _walk_remote(self, dirpath):
dirnames = []
filenames = []
for fd in self.sftp.listdir_attr(dirpath):
if S_ISDIR(fd.st_mode):
dirnames.append(fd.filename)
else:
filenames.append(fd.filename)
yield dirpath, dirnames, filenames
for dirname in dirnames:
new_dirpath = os.path.join(dirpath, dirname)
# yield from self._walk_remote(new_dirpath)
for walk in self._walk_remote(new_dirpath):
yield walk
def getall(self,local,remote):
st_mode = self.sftp.stat(remote).st_mode
if not S_ISDIR(st_mode):
filename = os.path.basename(remote)
self.sftp.get(remote, os.path.join(local, filename))
else:
parent, child = os.path.split(remote)
for dirpath, dirnames, filenames in self._walk_remote(remote):
dirpath = dirpath.replace(parent, '.')
parentc = os.path.join(local,dirpath)
if not os.path.exists(parentc):
os.makedirs(parentc)
for dirname in dirnames:
try:
os.makedirs(os.path.join(local, dirpath, dirname))
except:
pass
for filename in filenames:
localpath = os.path.join(local, dirpath, filename)
remotepath = os.path.join(parent, dirpath, filename)
self.sftp.get(remotepath, localpath)
self.scp.close()
if __name__=='__main__':
port = 22
now = datetime.datetime.now()
strdatetime = now.strftime("%Y-%m-%d")
year=strdatetime.split('-')
mon=strdatetime.split('-')
day=strdatetime.split('-')
Datenow1= year + "/" + mon + "/" + day + "/"
Datenow= year + "/" + mon
print "-"*50
f = file('/home/python/filelist','r')
c = f.readlines()
for x in c:
hostname = x.split('::')
password = x.split('::')
username = x.split('::')
local= x.split('::').strip('\n')
remotes = x.split('::').strip('\n')
localz=local + "/" + mon + "/" + day
if remotes.endswith('/'):
remote1 = remotes + Datenow
remote2 = remotes + Datenow1
else:
remote3 = remotes
if not os.path.exists(localz):
remote = remote1
getthread=get_thread(hostname,password,username)
getthread.getall(local,remote)
else:
remote = remote2
echo_cmd='/bin/find %s -maxdepth 1 -type d -mmin -5' % (remote)
cmd_thread=run_cmd(hostname,password,username,port,echo_cmd)
result=cmd_thread.run()
del result
for item in result:
print str(item)
items = item.strip('\n')
getthread=get_thread(hostname,password,username)
getthread.getall(localz,items)
f.close()
执行time python test.py最主要可以增量五分钟修改的，节约时间。
页:
[1]