python并行运行函数

uytre

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84

# -*- coding: utf-8 -*- import multiprocessing import os, time,random import pymysql curdir = os.path.dirname(__file__) def db_conn():     conn = pymysql.connect(host='localhost',user='root',password='123456',db='entries', charset='utf8')     #conn = pymysql.connect(host='localhost',user='root',password='root',db='db', charset='utf8', cursorclass=pymysql.cursors.DictCursor)     return conn def db_query(conn, sql):     cursor = conn.cursor()     cursor.execute(sql)     result = cursor.fetchall()     return result def fun_1():     fp = open(os.path.join(curdir, 'engines.txt'), 'w')     conn = db_conn()     sql = 'select engine,support,comment from engines;'     result = db_query(conn, sql)     for res in result:         fp.write("%s|%s|%s\n" %(res[0],res[1], res[2]))         fp.flush()     fp.close()     conn.close() def fun_2():     fp = open(os.path.join(curdir, 'collations.txt'), 'w')     conn = db_conn()     sql = 'select collation_name,character_set_name,id,is_default, is_compiled from collations;'     result = db_query(conn, sql)     for res in result:         fp.write("%s|%s|%s|%s|%s\n" %(res[0],res[1], str(res[2]),res[3],res[4]))         fp.flush()     fp.close()     conn.close() def fun_3():     fp = open(os.path.join(curdir, 'indexes.txt'), 'w')     conn = db_conn()     sql = 'select name,table_id,type,n_fields,page_no from indexes;'     result = db_query(conn, sql)     for res in result:         fp.write("%s|%s|%s|%s|%s\n" %(res[0],res[1], res[2],res[3],res[4]))         fp.flush()     fp.close()     conn.close() def main():     conn = db_conn()     fun_list = [ fun_1, fun_2, fun_3 ]     print("parent process %s" %  os.getpid())     pool = multiprocessing.Pool(3)     start = time.time()     for func in fun_list:         print("func name", func)         pool.apply_async(func)     print('waiting for all subprocess done...')     pool.close()     pool.join()     end = time.time()     print('All subprocess done, run %0.2f seconds' % (end - start))     conn.close() if __name__ == '__main__':     main()  # 0.3  - 0.42     '''     start = time.time()     conn = db_conn()     fun_1(conn)     fun_2(conn)     fun_3(conn)     conn.close()     end = time.time()     print('All done, run %0.2f seconds' % (end - start))  #6.61 6.94 7.03 7.30 6.91     '''

当fun_1, fun_2, fun_3本身不是很耗时的时候，并行的效率没有顺序执行的效率高。