| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 | #!/usr/bin/env python #-*- coding:UTF-8 -*- """ @Item : Rsync Backup @Author : Villiam Sheng @Group : Linux Group @Date : 2012-08-13 @Funtion: rsync_conf: Configuration rsync server, and the server #res[os.path.join(root,fpath)] = int((time.time() - os.stat(os.path.join(root,fpath)).st_ctime) / 86910) #int((time.time() - os.stat(os.path.join(root,fpath)).st_ctime) / 86910) #try: # res[os.path.join(root,fpath)] = time.strptime((fpath.split('_')[2]),'%Y%m%d%H%M%S').tm_yday #截取文件名,得到时间天数, #except Exception,e: # print e #res[os.path.join(root,fpath)] = int((time.time() - os.stat(os.path.join(root,fpath)).st_ctime) / 86910) Central idea: """ import os,sys,time,re,socket,threading,json,base64,traceback,ConfigParser,fcntl,struct from rsync_log import rsync_log from rsync_post import rsync_post from statvfs import F_BLOCKS,F_BAVAIL,F_BSIZE pcg = 0 """ 生成斐波那契数列""" lists = [] a,b = 0 , 1 while b < = 365 : a,b = b ,a + b lists.append(b) class rsync_thread(threading.Thread): def __init__( self ,path): threading.Thread.__init__( self ) self .log = rsync_log() self .path = path """ 计算当前磁盘的使用百分比""" def disk( self ): try : vfs = os.statvfs( self .path) disk_full = int (vfs[F_BLOCKS] * vfs[F_BSIZE] / 1024 / 1024 / 1024 ) disk_free = int (vfs[F_BAVAIL] * vfs[F_BSIZE] / 1024 / 1024 / 1024 ) return '%.1f' % ( float (disk_full - disk_free) / disk_full * 100 ) except : self .log.log_info( 'rsync_info.err' , 'dfile.disk' ,traceback.print_exc()) return traceback.print_exc() def run( self ): global pcg old_df = [] # 上一年的删除历史文件 new_df = [] # 今年的删除历史文件 sf = [] # 保留的历史文件 res = {} # 所有文件的天数及文件的路径 rs = 0 # 删除文件的总和 size = [] # 获取删除文件的大小 msize = [] # 今天备份所有文件的大小 tday_size = [] # 今天备份文件的大小 ms = 0 # 今天备份文件的总和 year = time.localtime().tm_year """ 得到文件的天数，以文件名作为key，天数作为value """ for root,dirs,files in os.walk( self .path): for fpath in files: res[os.path.join(root,fpath)] = time.localtime(os.stat(os.path.join(root,fpath)).st_ctime).tm_yday """ 判断文件的天数是否符合斐波那契数列，符合条件append到sf列表中,不符合append df列表中 """ for v,k in res.items(): if k in lists: sf.append(k) self .log.log_info( 'log_info.save' , 'dfile.rsync_thread' , '%s:::%s' % (v,k)) elif k not in lists: if year ! = time.localtime(os.stat(v).st_ctime).tm_year: old_df.append({v:k}) else : new_df.append({v:k}) """ try: for s in range(len(new_df)): for f,k in new_df[s].items(): tday_size.append(k) if max({}.fromkeys(tday_size).keys()) == k: msize.append(os.path.getsize(f)) except: pass """ c = [] pcg = float ( self .disk()) """ 判断今天是否有新的文件备份，在删除的列表中删除最后一天的数据，但必须保证磁盘的使用百分比大于 %55 """ if time.localtime().tm_yday in res.values(): if len (old_df) ! = 0 : for s in range ( len (old_df)): for f,k in old_df[s].items(): c.append(k) for s in range ( len (old_df)): for f,k in old_df[s].items(): if min ({}.fromkeys(c).keys()) = = k and pcg > 91 : size.append(os.path.getsize(f)) os.system( 'rm -frv %s' % f) self .log.log_info( 'log_info.delete' , 'remove cmd' , 'rm -frv %s %s' % (f,k)) elif pcg < = 91 : break pcg = float ( self .disk()) elif len (new_df) ! = 0 : for s in range ( len (new_df)): for f,k in new_df[s].items(): c.append(k) for s in range ( len (new_df)): for f,k in new_df[s].items(): if min ({}.fromkeys(c).keys()) = = k and pcg > 91 : size.append(os.path.getsize(f)) os.system( 'rm -frv %s' % f) self .log.log_info( 'log_info.delete' , 'remove cmd' , 'rm -frv %s %s' % (f,k)) elif pcg < = 91 : break pcg = float ( self .disk()) for s in size: rs + = s #for m in msize: # ms += m self .log.log_info( 'log_info.delete' , 'Disk release %s %s MB' % ( self .path,rs / 1024 / 1024 ), 'Disk append %s %s MB' % ( self .path,ms / 1024 / 1024 )) else : self .log.log_info( 'log_info.delete' , 'Disk files ' , ' %s No update file' % self .path) sys.exit() class rsync_dfile( object ): def __init__( self ): self .log = rsync_log() self .rsync_post = rsync_post() def work( self ): fp = open ( '/proc/mounts' , 'r' ) m_info = fp.readlines() fp.close() data = {} sections = [] for i in m_info: if i.find( 'data=ordered' ) ! = - 1 or i.find( 'mfs' ) ! = - 1 or i.find( 'nfs' ) ! = - 1 : if os.path.ismount( str (i.split()[ 1 ])): if str (i.split()[ 1 ]) ! = '/' : if str (i.split()[ 1 ]) ! = '/root' : if str (i.split()[ 1 ]) ! = '/var' : if len (i.split()[ 1 ]) ! = 1 : if not i.find( 'sunrpc' ) ! = - 1 : rs_thread = rsync_thread(i.split()[ 1 ]) rs_thread.start() while threading.active_count() > 1 : time.sleep( 1 ) conf = ConfigParser.ConfigParser() conf.read( '/etc/rsyncd.conf' ) try : for i in conf.sections(): if i ! = 'global' : sections.append(i) for i in sections: vfs = os.statvfs(conf.get(i, 'path' )) disk_full = int (vfs[F_BLOCKS] * vfs[F_BSIZE] / 1024 / 1024 / 1024 ) disk_free = int (vfs[F_BAVAIL] * vfs[F_BSIZE] / 1024 / 1024 / 1024 ) s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) try : ip = socket.inet_ntoa(fcntl.ioctl(s.fileno(), 0x8915 ,struct.pack( '24s' , 'eth0' ))[ 20 : 24 ]) except : ip = socket.inet_ntoa(fcntl.ioctl(s.fileno(), 0x8915 ,struct.pack( '24s' , 'eth1' ))[ 20 : 24 ]) t_info = { 'flag' : 0 , 'store_ip' :ip, 'store_module' :i, 'store_path' :conf.get(i, 'path' ), 'disk_full' :disk_full, 'disk_free' :disk_free, 'action' : 'rsync_renew' } data[ 'param' ] = base64.b64encode(json.dumps(t_info)) self .rsync_post.work(data) self .log.log_info( 'rsync_info.err' , 'dfile.work' ,t_info) except Exception,e: t_info = { 'flag' : 1 , 'store_ip' :ip, 'store_module' :i, 'store_path' :conf.get(i, 'path' ), 'disk_full' :disk_full, 'disk_free' :disk_free, 'action' : 'rsync_renew' } data[ 'param' ] = base64.b64encode(json.dumps(t_info)) self .rsync_post.work(data) self .log.log_info( 'rsync_info.err' , 'dfile.work' ,e) if __name__ = = "__main__" : rs = rsync_dfile() while True : rs.work() if pcg < = 91 : break |