import ctypes import datetime import os import platform import shutil import sys import traceback import s3upload def freespace(p): """ FROM: http://atlee.ca/blog/2008/02/23/getting-free-diskspace-in-python/ http://stackoverflow.com/questions/51658/cross-platform-space-remaining-on-volume-using-python Returns the number of free bytes on the drive that ``p`` is on """ if platform.system() == 'Windows': free_bytes = ctypes.c_ulonglong(0) ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(p), None, None, ctypes.pointer(free_bytes)) return free_bytes.value else: s = os.statvfs(p) return s.f_bsize * s.f_bavail def directory_size(path): """ FROM: http://stackoverflow.com/questions/1392413/calculating-a-directory-size-using-python """ total_size = 0.0 for dirpath, dirnames, filenames in os.walk(path): for f in filenames: fp = os.path.join(dirpath, f) total_size += os.path.getsize(fp) return total_size def parse_date_dir(d, date_fmt): if not os.path.exists(d): raise ValueError('%s does not exist' % d) if not os.path.isdir(d): raise ValueError('%s is not a directory' % d) return datetime.datetime.strptime(os.path.basename(d), date_fmt) def is_date_dir(d, date_fmt): try: parse_date_dir(d, date_fmt) return True except ValueError: return False def get_directory_list(path, date_fmt='%Y%m%d'): parse_func = lambda d: parse_date_dir(d, date_fmt) subdirs = [os.path.join(path, child) for child in os.listdir(path)] datedirs = [d for d in subdirs if is_date_dir(d, date_fmt)] return sorted(datedirs, key=parse_func) def upload_dirs_until_free(path, target_free_mbs, dryrun): starting_freespace = float(freespace(path)) dirs_uploaded = 0 files_uploaded = 0 cur_freespace = starting_freespace reclaimed_space = 0.0 error = False directory_list = get_directory_list(path) if not directory_list: print "Target directory %s has no subdirectories!" % path sys.exit(1) print "Target directory: %s" % path print "Starting freespace: %.02f MBs" % (starting_freespace / 1024 / 1024) print "Target freespace: %.02f MBs" % target_free_mbs print "Image subdirectories: %s" % len(directory_list) if dryrun: print print '!!! Doing dryrun -- current free space will be estimated !!!' print try: for dir_to_upload in directory_list: if cur_freespace >= target_free_mbs * 1024 * 1024: break dir_size = directory_size(dir_to_upload) print 'Uploading %s (%.02f MBs)' % (dir_to_upload, dir_size / 1024 / 1024) res = s3upload.do_upload(dir_to_upload, verbose=False, dryrun=dryrun) files_uploaded += res['files_uploaded'] print "%s files uploaded in %.02fs" % (res['files_uploaded'], res['sec_elapsed']) dirs_uploaded += 1 reclaimed_space += dir_size if not dryrun: print "Deleting %s" % dir_to_upload shutil.rmtree(dir_to_upload) if dryrun: cur_freespace += dir_size else: cur_freespace = float(freespace(path)) print "%.02f MBs now free" % (cur_freespace / 1024 / 1024) print except Exception: print "An unexpected error occured!" error = True traceback.print_exc() print "---------------------------------------" if error: pass else: pass print "Finished successfully" if not error else "!!! Terminated abnormally !!!" print "Current free space: %.02f MBs" % (cur_freespace / 1024 / 1024) print "Reclaimed space: %.02f MBs" % (reclaimed_space / 1024 / 1024) print "Directories uploaded: %s" % dirs_uploaded print "Files uploaded: %s" % files_uploaded if __name__ == '__main__': if not 4 <= len(sys.argv) <= 5: print "usage: dailyimgupload.py workingdir path megabytes [dryrun]" sys.exit(1) wd = sys.argv[1] if not os.path.isdir(wd): print "Invalid working directory: %s" % wd sys.exit(1) print "Switching working directory to %s" % wd os.chdir(wd) path = sys.argv[2] if not os.path.isdir(path): print "invalid image directory: %s" % path sys.exit(1) mbs = sys.argv[3] try: target_free_mbs = float(mbs) except ValueError: print "invalid number of megabytes: %s" % mbs sys.exit(1) if len(sys.argv) == 5: dryrun = sys.argv[4] if dryrun in ('true', 'false'): dryrun = dryrun == 'true' else: print "invalid dry run argument: %s (must be either 'true' or 'false')" % dryrun sys.exit(1) else: dryrun = True upload_dirs_until_free(path, target_free_mbs, dryrun)