1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
| #!/usr/bin/python
# -*- coding:utf-8 -*-
_author__ = 'mujibin'
#import python lib
import random
import urllib
import datetime
import time
import MySQLdb
import os
import time
import re
import urllib2
import json
import string
import sys
import time
import paramiko
#add path
sys.path.append("/data1/salt/mysqlapi/salt/")
#import salt repaire function
from multiprocessing import *
import logging
from salt_minion_list import *
from init_server import *
from check_salt import *
#from check_salt_bak import *
from salt_repair_ssh import *
reload(sys)
sys.setdefaultencoding('utf8')
H3303='*****.cn'
H3304m='******.cn'
P3303=3303
P3304=3304
dp_admin='dp_admin'
HOST_PORT='3303'
HOST_USER = 'mysqlha'
HOST_PASSED = '********'
db='test'
port='*******'
c_date = time.strftime("%Y%m%d",time.localtime())
c_time = time.strftime("%Y-%m-%d %H:%M:%S",time.localtime())
'''
log_path = "/data1/dbatemp/salt/logs"
is_path=os.path.exists(log_path)
if not is_path:
os.makedirs(log_path)
log_name = "salt_reparie.log"
logger = logging.getLogger()
handler = logging.FileHandler(os.path.join(log_path,log_name))
formater = logging.Formatter("%(asctime)s %(levelname)s [%(funcName)s :%(lineno)d] %(message)s")
handler.setFormatter(formater)
logger.addHandler(handler)
logger.setLevel(logging.NOTSET)
#logger.setLevel(logging.INFO)
#logger.setLevel(logging.DEBUG)
#logger.setLevel(logging.ERROR)
'''
##########################################################
salt_yes = datetime.date.today()
##########################################################
#ssh api argument
method = "sync"
output = "json"
ignore_error = "true"
timeout = "28"
##########################################################
slat_minion_check_CONSTANT="salt-minion"
##########################################################
SALT = "salt"
VERSION = "5.3"
###########################################################
#master dns transfor to ip
###########################################################
def getIp(domain):
import socket
myaddr = socket.getaddrinfo(domain,'http')[0][4][0]
return myaddr
MASTERDNS= "******.cn"
MASTERIP = getIp(MASTERDNS)
##########################################################
def ssh_connect_bak(host):
client = paramiko.SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
privatekeyfile = os.path.expanduser('/root/.ssh/id_rsa')
mykey = paramiko.RSAKey.from_private_key_file(privatekeyfile)
host=host.strip()
client.connect(host,26387,username='root',timeout=2,pkey=mykey)
return client
def ssh_connect(host):
client = paramiko.SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
host=host.strip()
client.connect(host,26387,username='root',timeout=10)
return client
def ssh_cmd(host,cmd):
try:
client = ssh_connect(host)
i,o,e = client.exec_command(cmd)
res = o.read().strip()
return res
except Exception,e:
msg = "The host:%s and cmd:%s execute exception." % (host,cmd)
#print msg
pass
def ssh_cmd_check(host,cmd1): #用来判断是否可以ssh登陆成功
flag=0
#host_info=[host,flag]
cmds=[cmd1]
try:
for cmd in cmds:
#paramiko.util.log_to_file('paramiko.log')
s = paramiko.SSHClient()
s.load_system_host_keys()
s.set_missing_host_key_policy(paramiko.AutoAddPolicy())
host=host.strip()
s.connect(host,26387,username='root',timeout=20)
s0,s1,s2 = s.exec_command(cmd1)
info = s1.read().strip()
#print s2.read().strip()
#host_info.append(info)
s.close()
flag=0
except Exception,e:
#根据第二个标志位确定ssh是否通
flag=1
#host_info[1]=-1
return flag
def run_cmd(ips,cmd,method,output,ignore_error,timeout):##这是公司的通道机，可以获取json格式的返回数据
_ips_ = ips
_cmd_ = cmd
#logger.debug(_cmd_)
_method_ = method
_output_ = output
_ignore_error_ = ignore_error
_timeout_ = timeout
_user_='***'
_key_='*****'
url='*****p.php'
argument={ 'user':_user_,'method':_method_,'output':_output_,'ignore_error':_ignore_error_,'key':_key_,'timeout':_timeout_,'ip':_ips_,'cmd':_cmd_}
try:
data = urllib.urlencode(argument)
response = urllib2.urlopen(url, data)
except Exception,e:
msg = "Call the api function error!"
pass
return response.read()
def select_in_3303(sql,host,user,port,passwd,db): ##查询sql
try:
db = MySQLdb.connect(host=host,user=user,port=port,passwd=passwd,db=db,connect_timeout=5,charset = "utf8")
cursor = db.cursor()
cursor.execute(sql)
rows = cursor.fetchall()
array = []
for row in rows:
array.append(str(row[0]))
db.close()
return array
except Exception,e:
##print str(e)
return []
##插入sql，因为还没有写入库的函数，所以没有用
def sql_insert(sql, port=3304, domain='*****', db='*****'):
try:
db = MySQLdb.connect(host=domain,user=HOST_USER,port=port,passwd=HOST_PASSED,db='swordfish',connect_timeout=3,charset="utf8")
cursor = db.cursor()
cursor.execute(sql)
db.commit()
db.close()
except Exception,e:
#print str(e)
db.rollback()
db.close()
##获取所有客户端有问题的服务器ip列表
def fix_list_salt():
sshList=[]
try:
saltsql="select ip_in from salt_mon where salt_mon_value != 'ok' and salt_mon_info not like '%None%' and ctime = (select ctime from salt_mon order by ctime desc limit 1);"
sshList=select_in_3303(sql=saltsql,host=H3304m,user=HOST_USER,port=P3304,passwd=HOST_PASSED,db='swordfish')
return sshList
except Exception,e:
print e
#判断salt是否安装，通过命令去判断，如果系统无salt，那么改命名则什么都不会
#返回；如果存在，则会返回该系统salt的版本。返回：0 表示系统上存在salt，但进程不一定起来；返回1
#表示，希望不存在salt。
def salt_exist_check(host):
try:
versionCmd = "rpm -qa | grep salt | wc -l"
#versionRes = run_cmd(host, versionCmd, method="sync",output="text",ignore_error="true",timeout=5)
versionRes = ssh_cmd(host,versionCmd)
#logger.info(host+":"+versionRes)
if int(versionRes) == 0:
status = 0
else:
status = 1
res = status
return res
except Exception,e:
msg = "The function salt_exist_check execute failed with host:%s" % host
#logger.error(msg)
#logger.error(msg)
#该函数尝试restart salt minion 客户端，在重启minion客户端之前，首先通过接口去master上
#删除该系统id的key,然后再将本地的key删除，最后重启。重启后通过判断salt进程是否存在，以此
#表明salt是否重启成功。返回0表示重启salt成功，返回1表示重启失败。
def salt_minion_restart(host):
"""
when salt minion installed, which will be
restart. This function remove the key of minion.
"""
try:
#logger.info("%s Try to restart the salt minion,this action can't guarante for success!" % host)
#salt_remove_key(host)
Cmd1 = """sudo rm -f /etc/salt/pki/minion/minion_master.pub"""
Cmd2 = """sudo /etc/init.d/salt-minion restart"""
#logger.info(host+" : "+rmKeyCmd)
#logger.info(host+" : "+startCmd)
rmRes1 = run_cmd(host, Cmd1, method="sync",output="text",ignore_error="true",timeout=10)
time.sleep(5)
rmRes2=run_cmd(host, Cmd2, method="sync",output="text",ignore_error="true",timeout=10)
#logger.info(host+" : "+rmRes)
#logger.info(host+" : "+startRes)
time.sleep(5)
saltExistStatus = salt_check(host)
if saltExistStatus == 0:
msg = 0
else:
msg = 1
res = msg
return res
except Exception,e:
msg = "The host:%s restart minion failed!" %(host)
#logger.error(msg)
#logger.error(e)
#该函数会自动删除系统安装的salt程序,包括salt与salt minion。如果返回0，表示删除成功；如果返回
#1，表示删除失败。
def remove_salt_minion(host):
try:
#logger.info("%s Try to remove salt minion!" % host)
versionCmd = "sudo rpm -qa | grep salt| grep -v grep"
versionRes = run_cmd(host, versionCmd, method="sync",output="json",ignore_error="true",timeout=10)
#versionRes = ssh_cmd(host,versionCmd)
verResJsion = json.loads(versionRes)
saltList = json.loads(verResJsion["RETURN"])['result'].split('\n')
ssh_cmd(host,'/etc/init.d/salt-minion stop > /dev/null 2>&1 ')
if len(saltList) > 1:
for one in range(len(saltList)-1):
rmCmd ="sudo yum remove -y %s > /dev/null 2>&1 " % (saltList[one])
#logger.info(host+" : "+rmCmd)
rmRes = ssh_cmd(host,rmCmd)
time.sleep(4)
print rmRes
#logger.info(host+" : "+rmRes)
else:
#logger.info("salt minion don't install!")
pass
versionStatus = salt_exist_check(host)
if versionStatus == 0:
status = 0
else:
status =1
res = status
print 'res:%s' %res
return res
except Exception,e:
msg = "The function remove_salt_minion_qa execute failed with host:%s" % host
#logger.info(msg)
#logger.info(e)
#该函数去判断系统的yum列表是否存在所需安装的salt版本。如果存在，则返回0；反之，则返回1。
def yum_check(host):
try:
#logger.info("%s Try to check yum." % host)
checkCmd = "sudo yum list | grep salt | grep 2015 | wc -l"
checkRes = ssh_cmd(host,checkCmd)
if checkRes != 0:
status = 0
else:
status = 1
msg = status
return msg
except Exception,e:
msg = "The host:%s check the yum error!" %(host)
#logger.error(msg)
#logger.error(e)
#该函数修复系统的yum源。修复成功，返回0；修复失败，返回1，就是一个初始化yum源的过程。
def yum_repaire(host):
try:
yumCmd1=""" ([ `ps -ef | grep yum | grep -v grep | wc -l` -ne 0 ] && sudo ps -ef | grep '/usr/bin/yum' | grep -v grep | awk '{print $2}' | xargs kill -9 || echo '') && (cd /var/lib/rpm/ && sudo rm -f __db.00*) && (sudo rpm --rebuilddb) && (sudo yum clean all) && (sudo chattr -i /etc/yum.conf) && (sudo echo 'include=http://****/conf/yumconf.php' > /etc/yum.conf) && (sudo rm -rf /etc/yum.repos.d/*) && (sudo yum -y remove ****dbp > /dev/null 2>&1) && (sudo yum -y install ****dbp > /dev/null 2>&1)
"""
ret1 = ssh_cmd(host,yumCmd1)
time.sleep(60)
if yum_check(host) == 0:
msg = 0
else:
msg = 1
status = msg
return msg
except Exception,e:
msg = "The host:%s try to repaire yum failed!" %(host)
#logger.error(msg)
#logger.error(msg)
#该函数去判断系统是否存在salt进程，如果存在则，返回0;反之，则返回1.
def salt_check(host):
try:
#logger.info("%s Check the process of salt." % host)
checkCmd = "ps -ef | grep salt-minion | grep -v grep | wc -l"
checkRes = ssh_cmd(host,checkCmd)
#pattern = re.compile(r".*salt")
#match = pattern.match(checkRes)
if checkRes != 0:
status = 0
else:
status = 1
msg = status
return msg
except Exception,e:
msg = "The host:%s salt check error!" %(host)
#logger.error(msg)
#logger.error(msg)
#该函数安装salt minion客户端，如果安装成功，返回0；反之，则返回1.
def install_salt_minion(host):
try:
#logger.info("Install salt minion.")
inSaltCmd = """([ `ps -ef | grep yum | grep -v grep | wc -l` -ne 0 ] && sudo ps -ef | grep '/usr/bin/yum' | grep -v grep | awk '{print $2}' | xargs kill -9 || echo '') && (sudo yum clean all) && (sudo yum -y install salt.noarch salt-minion.noarch)"""
#in1Res = run_cmd(host, inSaltCmd, method, output, ignore_error, timeout)
in1Res = ssh_cmd(host,inSaltCmd)
#logger.info(host+" : "+in1Res)
#print in1Res
time.sleep(20)
saltInStatus = salt_exist_check(host)
if int(saltInStatus) == 1:
status = 0
else:
status = 1
res = status
return res
except Exception,e:
msg = "The host:%s install minion failed!" %(host)
#logger.debug(msg)
#logger.error(e)
##该函数检测服务器ip是否能ping通
def ping_mon_by_host(host):
try:
ping_cmd = "ping -c 1 -w 2 %s > /dev/null" % host
ret = os.system(ping_cmd)
if ret == 0:
status = 0
msg = "The host %s ping ok" % host
else:
status = 1
msg = "The host %s ping failed" % host
result = status
return result
except Exception,e:
msg = """The host %d: ping_mon_by_host failed!""" % host
#logger.error(msg)
#logger.error(e)
#检查master与minion端通信是否成功
def check_salt_minion(host):
try:
cmd = "salt '%s' -t 7 cmd.run 'uptime'" %host
ret = ssh_cmd(MASTERIP,cmd)
msg = ""
if ret and 'load' in ret:
status = 0
msg = 'ok'
else :
status = 1
try:
msg = ret.split(':')[1].strip()
except Exception,e:
msg = ret
result = {'status':status,'message':msg}
return result
except Exception,e:
pass
#该函数检测nfs挂载状态
def nfs_check(host):
mount_number_cmd = "mount | grep 'knfs'| wc -l"
mount_number = ssh_cmd(host,mount_number_cmd)
if int(mount_number) != 0:
mount_data_cmd = "mount | grep 'knfs' | awk -F ' ' '{print $3}'"
umount = run_cmd(host, mount_data_cmd, method="sync",output="json",ignore_error="true",timeout=10)
umJsion = json.loads(umount)
dataList = json.loads(umJsion["RETURN"])['result'].split('\n')
for one in range(len(dataList)-1):
rmCmd ="umount -l %s > /dev/null 2>&1 " % (dataList[one])
rmRes = ssh_cmd(host,rmCmd)
time.sleep(2)
mount_number_cmd2 = "mount | grep 'knfs'| wc -l"
mount_number2 = ssh_cmd(host,mount_number_cmd2)
if int(mount_number2) != 0:
msg = 1
else:
msg = 0
else:
msg = 0
return msg
#自动修复salt主程序
def salt_repaire(host):
try:
msg = ""
pingStatus = ping_mon_by_host(host)
if pingStatus == 0:
#判断是否可以登录
sshStatus=ssh_cmd_check(host,'date')
if sshStatus == 0:
#监测nfs挂载是否正常
nfsStatus = nfs_check(host)
if nfsStatus == 0:
#修复yum源
print 'yum_repair'
yumStatus = yum_repaire(host)
#print yumStatus
if yumStatus == 0:
#卸载salt minion客户端
print 'remove salt'
removeStatus = remove_salt_minion(host)
if removeStatus == 0:
print 'install salt'
#安装salt minion客户端
installStatus = install_salt_minion(host)
if installStatus == 0:
#启动salt minion 客户端
print 'start salt'
restartStatus = salt_minion_restart(host)
if restartStatus == 0:
print 'master-minion check'
minionStatus = check_salt_minion(host)
if minionStatus["status"] == 0:
print '%s:ok' % host
else:
print '%s:%s' %(host,minionStatus["message"])
else:
msg = "%s:salt minion restart error!" % host
else:
msg = "%s:install salt minion failed!" % host
else:
msg = "%s:remove salt minion failed!" % host
else:
msg = "%s: yum occur error!" % host
else:
msg = "%s:nfs err" %host
else:
msg = "%s: bad ssh,go failed!" % host
else:
msg = "%s: The host can not ping!" % host
print msg
#info = msg
#re_info = msg
#return info
#相关信息入库
#ping_status = p_status
#salt_status = s_status
#salt_minion_mon(host,ping_status,salt_status,re_info)
#salt_info(host,info)
return info
except Exception,e:
msg = "Salt repaire failed with host:%s " % host
#logger.info(msg)
#logger.info(e)
def scheduler_repaire():
minionList = fix_list_salt()
pool = Pool(8)
pool.map(salt_repaire,minionList)
pool.close()
pool.join()
if __name__ == "__main__":
scheduler_repaire()
|
QQ群⑧：
窥视卡
雷达卡
发表于 2015-11-12 09:00:53
提升卡
置顶卡
沉默卡
喧嚣卡
变色卡
千斤顶
显身卡