pythonマルチスレッドダウンロードossピクチャ

2261 ワード

しんそくが通る
 
  
import urllib2,os
from HTMLParser import HTMLParser
import  urllib
import  oss2
import  MySQLdb
import socket
from multiprocessing import Pool
import  traceback
from multiprocessing.dummy import Pool as ThreadPool

def save_pic(url,filename):
    count=0
    while True:
        try:
            print filename,'begin...'
            urllib.urlretrieve(url, filename)
        except socket.timeout:
            print filename,'count:',count
            count+=1
        except Exception,e:
            print filename,'other',e
            count+=1
        else:
            print filename,'successfully!'
            break

if __name__ == "__main__":

    access_key_id = os.getenv('OSS_TEST_ACCESS_KEY_ID', '**********')
    access_key_secret = os.getenv('OSS_TEST_ACCESS_KEY_SECRET', '**************')
    bucket_name = os.getenv('OSS_TEST_BUCKET', '**********')
    endpoint = os.getenv('OSS_TEST_ENDPOINT', '*************')

    host = 'localhost'
    user = 'root'
    pwd = 'root'
    db = 'test'
    conn = MySQLdb.connect(host,user,pwd,db,charset='utf8')
    cursor = conn.cursor()

    sql='select * from ks_table'

    img_dir = "D:\\images"

    if not os.path.isdir(img_dir):
        os.mkdir(img_dir)

    os.chdir(img_dir)
    print os.getcwd()
    url = ""

    try:
       # execute sql
       cursor.execute(sql)
       # fetch records

       results = cursor.fetchall()
       print 'Parent process %s.' % os.getpid()
       pool=ThreadPool(50)
       for row in results:
          xuexinid = row[0]
          questionurl=row[1]
          url = "http://"+questionurl.replace("@", ".*******.aliyuncs.com/");
          filename = os.path.basename(url)
          filename=xuexinid+'_'+filename

          pool.apply_async(save_pic,args=(url,filename))
       pool.close()
       pool.join()
       print 'all successfully!'
    except Exception,e:
       print "Error: unable to fecth data",e.message
       print  traceback.print_exc()


    cursor.close()
    conn.close()