MySQLバックアップツールmydumperバックアップ

#!/usr/bin/phthon


import os
import time
import commands
import shutil
import threading
from os.path import join, getsize
import MySQLdb as mydb


#     
baseDir = "/data2/backup/backup_data/"


# ns   wx;          (mydumper       )，    True，   False.
idc = 'ns'; isZip = True


#          ，True    ,      False, retry_sleep         ( )
is_errRetryBackup = True; retry_sleep = 300


#     
backup_date = time.strftime("%Y%m%d")


#     
cmd  = "/usr/local/bin/mydumper -h %s -u root -p password? -P %s %s -t 5 -o %s"


'''
    ：
  1. mydumper       ,            
  2.            (mydumper     )
  3.              
  4.          
'''
def main():
    thread_pool = []


    #       
    zip = '-c' if isZip == True else ''


    #         ip,   ,   ,        
    #f = open('/data2/backup/cnf/other_list.cnf', 'r')
    f = open('/data/other_list.cnf', 'r')
    for lines in f.readlines():
        if (not lines.startswith('#') and len(lines.strip()) > 0):
            str = lines.split()
            host, businessName, port, isMaster = str[0], str[1], str[2], str[3]
            #            
            dir = baseDir + '/' + businessName;
            if (not os.path.exists(dir)):
                 os.makedirs(dir)


            dir += "/%s%s" % (businessName, backup_date)
            #     : dir ,     : dir/name+    
            strcmd  = cmd % (host, port, zip, dir)


            th = threading.Thread(target = mydumper, args =(strcmd, dir, businessName, host, port, is_errRetryBackup, int(isMaster)))
            thread_pool.append(th)


    if (thread_pool):
        for t in thread_pool:
            t.daemon = True
            t.start()
        for t in thread_pool:
            t.join()


def mydumper(sCmd, backupDir, businessName, host, port, is_Retry, isMaster):
    master_host = ""; backup_host = host; name = businessName; port = port; backup_type = 1; file = "";
    start_time = ""; stop_time = ""; returncode = 0; file_size = 0; slave_statement = ""; std_err = "";


    start_time = time.strftime("%Y%m%d%H%M%S")


    #          
    if (os.path.exists(backupDir)):
        shutil.rmtree(backupDir)


    #     
    returncode, std_err = execute(sCmd)


    stop_time = time.strftime("%Y%m%d%H%M%S")


    if (returncode == 0):
        #    std_err           。
        if (std_err.strip() != ""):
            returncode = 123456
        else:
            #    change master to   ，          
            returncode, std_err, master_host, slave_statement = statement(backupDir, backup_host, isMaster)


            if (returncode == 0):
                file = backupDir


    if (returncode != 0):
        #        :    + _ERR
        errDir = backupDir + "_ERR"
        os.rename(backupDir, errDir)
        file = errDir


    #       
    file_size = getDirsize(file)


    if (len(std_err) > 255):
        std_err = std_err[:250] + "..."


    my_args  = [idc, master_host, backup_host, name, port, backup_type, file, start_time, stop_time, returncode, file_size, slave_statement, std_err]


    #      
    call_proc(my_args)


    #           ?       .
    if (is_Retry == True and returncode != 0):
        time.sleep(retry_sleep)


        oldfile = sCmd.split('-o')[1]
        pos = oldfile.rfind("/") + 1


        #        ,             
        retry_file = oldfile[:pos] + "ReBackup-" + oldfile[pos:]


        retryCmd = sCmd.replace(oldfile,  retry_file)


        #     
        mydumper(retryCmd, retry_file.strip(), name, host, port, False, isMaster)


def getDirsize(path):
    #          
    size = 0L
    for root, dirs, files in os.walk(path):
        size += sum([getsize(join(root, name)) for name in files])
    return (size)


def statement(path, backup_host, isMaster):
    '''
      :   metadata   change master to   
    1.     :     : metadata.partial,    metadata.partial    : metadata          
    2. metadata  3 :
       (1) Started dump:       .
       (2) master  log-file   log-pos    (  ); slave  host、log-file   log-pos    （   slave   )
       (3) Finished dump:       
    3.     , master_host   change master to   
    '''
    path += "/metadata"; sMetadata = ""; master_host = ""; er_code = 654321; er_info = "%s not exists !!!" % (path)


    if (os.path.exists(path)):
        if (isMaster != 1):
            #     slave
            num = 3
            sFinds = "SLAVE STATUS"
        else:
            num = 2
            sFinds = "MASTER STATUS"


        f = open(path, 'r')
        rows = f.readlines(); i = 100; lst =[]
        for s in rows:
            if (s.find(sFinds) > 0):
                i = 1; continue


            if (i <= num):
                lst.append(s.split(':')[1].strip())
                i += 1


        if (isMaster == 1):
            #     master
            master_host = backup_host
            log_file, log_pos = lst;
        else:
            #     slave
            master_host, log_file, log_pos = lst;


        er_code = 0
        er_info = ""
        sMetadata = "CHANGE MASTER TO MASTER_HOST='%s',MASTER_LOG_FILE='%s',MASTER_LOG_POS=%s,MASTER_USER='rep_user',MASTER_PASSWORD='meizu.com'" % (master_host, log_file, log_pos )


    return (er_code, er_info, master_host, sMetadata)


def execute(cmd):
    '''
    1.   shell   
    2.       (returncode = 0      , std_err         )
    '''
    try:
        returncode, std_err = commands.getstatusoutput(cmd)
        return (returncode, std_err)
    except os.error, e:
        #      1001   
        return (1001, e)


def call_proc(my_args):
    #          
    try:
        conn = mydb.connect(host = '127.0.0.1', user = 'test', passwd = 'zxc/213?', db = 'meizu_item')
        cur  = conn.cursor()


        cur.callproc('sp_backup_i',[my_args[0], my_args[1], my_args[2], my_args[3], my_args[4], my_args[5], my_args[6], my_args[7], my_args[8], my_args[9], my_args[10], my_args[11], my_args[12]])
        conn.commit()
    except mydb.Error, e:
        pass
        # print "Mysql Error %d: %s" % (e.args[0], e.args[1])
    finally:
        cur.close(); conn.close()


if __name__ == '__main__':
    main()