python並列実行関数
2644 ワード
# -*- coding: utf-8 -*-
import multiprocessing
import os, time,random
import pymysql
curdir = os.path.dirname(__file__)
def db_conn():
conn = pymysql.connect(host='localhost',user='root',password='123456',db='entries', charset='utf8')
#conn = pymysql.connect(host='localhost',user='root',password='root',db='db', charset='utf8', cursorclass=pymysql.cursors.DictCursor)
return conn
def db_query(conn, sql):
cursor = conn.cursor()
cursor.execute(sql)
result = cursor.fetchall()
return result
def fun_1():
fp = open(os.path.join(curdir, 'engines.txt'), 'w')
conn = db_conn()
sql = 'select engine,support,comment from engines;'
result = db_query(conn, sql)
for res in result:
fp.write("%s|%s|%s
" %(res[0],res[1], res[2]))
fp.flush()
fp.close()
conn.close()
def fun_2():
fp = open(os.path.join(curdir, 'collations.txt'), 'w')
conn = db_conn()
sql = 'select collation_name,character_set_name,id,is_default, is_compiled from collations;'
result = db_query(conn, sql)
for res in result:
fp.write("%s|%s|%s|%s|%s
" %(res[0],res[1], str(res[2]),res[3],res[4]))
fp.flush()
fp.close()
conn.close()
def fun_3():
fp = open(os.path.join(curdir, 'indexes.txt'), 'w')
conn = db_conn()
sql = 'select name,table_id,type,n_fields,page_no from indexes;'
result = db_query(conn, sql)
for res in result:
fp.write("%s|%s|%s|%s|%s
" %(res[0],res[1], res[2],res[3],res[4]))
fp.flush()
fp.close()
conn.close()
def main():
conn = db_conn()
fun_list = [ fun_1, fun_2, fun_3 ]
print("parent process %s" % os.getpid())
pool = multiprocessing.Pool(3)
start = time.time()
for func in fun_list:
print("func name", func)
pool.apply_async(func)
print('waiting for all subprocess done...')
pool.close()
pool.join()
end = time.time()
print('All subprocess done, run %0.2f seconds' % (end - start))
conn.close()
if __name__ == '__main__':
main() # 0.3 - 0.42
'''
start = time.time()
conn = db_conn()
fun_1(conn)
fun_2(conn)
fun_3(conn)
conn.close()
end = time.time()
print('All done, run %0.2f seconds' % (end - start)) #6.61 6.94 7.03 7.30 6.91
'''
fun_1, fun_2, fun_3自体があまり時間がかからない場合は,並列の効率が順次実行されない効率が高い.