pythonマルチスレッドダウンロードossピクチャ
2261 ワード
しんそくが通る
import urllib2,os from HTMLParser import HTMLParser import urllib import oss2 import MySQLdb import socket from multiprocessing import Pool import traceback from multiprocessing.dummy import Pool as ThreadPool def save_pic(url,filename): count=0 while True: try: print filename,'begin...' urllib.urlretrieve(url, filename) except socket.timeout: print filename,'count:',count count+=1 except Exception,e: print filename,'other',e count+=1 else: print filename,'successfully!' break if __name__ == "__main__": access_key_id = os.getenv('OSS_TEST_ACCESS_KEY_ID', '**********') access_key_secret = os.getenv('OSS_TEST_ACCESS_KEY_SECRET', '**************') bucket_name = os.getenv('OSS_TEST_BUCKET', '**********') endpoint = os.getenv('OSS_TEST_ENDPOINT', '*************') host = 'localhost' user = 'root' pwd = 'root' db = 'test' conn = MySQLdb.connect(host,user,pwd,db,charset='utf8') cursor = conn.cursor() sql='select * from ks_table' img_dir = "D:\\images" if not os.path.isdir(img_dir): os.mkdir(img_dir) os.chdir(img_dir) print os.getcwd() url = "" try: # execute sql cursor.execute(sql) # fetch records results = cursor.fetchall() print 'Parent process %s.' % os.getpid() pool=ThreadPool(50) for row in results: xuexinid = row[0] questionurl=row[1] url = "http://"+questionurl.replace("@", ".*******.aliyuncs.com/"); filename = os.path.basename(url) filename=xuexinid+'_'+filename pool.apply_async(save_pic,args=(url,filename)) pool.close() pool.join() print 'all successfully!' except Exception,e: print "Error: unable to fecth data",e.message print traceback.print_exc() cursor.close() conn.close()