From d765ecafa41542f3745522c164f9c8ed9bb0eb62 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Sat, 6 Nov 2010 20:29:40 -0400 Subject: Added dailyimgupload.py, updated s3upload.py --- scripts/dailyimgupload.py | 152 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 scripts/dailyimgupload.py (limited to 'scripts/dailyimgupload.py') diff --git a/scripts/dailyimgupload.py b/scripts/dailyimgupload.py new file mode 100644 index 0000000..81ee44a --- /dev/null +++ b/scripts/dailyimgupload.py @@ -0,0 +1,152 @@ +import ctypes +import datetime +import os +import platform +import sys +import traceback +import s3upload + + +def freespace(p): + """ + FROM: http://atlee.ca/blog/2008/02/23/getting-free-diskspace-in-python/ + http://stackoverflow.com/questions/51658/cross-platform-space-remaining-on-volume-using-python + Returns the number of free bytes on the drive that ``p`` is on + """ + if platform.system() == 'Windows': + free_bytes = ctypes.c_ulonglong(0) + ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(p), None, None, ctypes.pointer(free_bytes)) + return free_bytes.value + else: + s = os.statvfs(p) + return s.f_bsize * s.f_bavail + +def directory_size(path): + """ + FROM: http://stackoverflow.com/questions/1392413/calculating-a-directory-size-using-python + """ + total_size = 0 + for dirpath, dirnames, filenames in os.walk(path): + for f in filenames: + fp = os.path.join(dirpath, f) + total_size += os.path.getsize(fp) + return total_size + +def parse_date_dir(d, date_fmt): + if not os.path.exists(d): + raise ValueError('%s does not exist' % d) + if not os.path.isdir(d): + raise ValueError('%s is not a directory' % d) + return datetime.datetime.strptime(os.path.basename(d), date_fmt) + +def is_date_dir(d, date_fmt): + try: + parse_date_dir(d, date_fmt) + return True + except ValueError: + return False + +def get_directory_list(path, date_fmt='%Y%m%d'): + parse_func = lambda d: parse_date_dir(d, date_fmt) + subdirs = [os.path.join(path, child) for child in os.listdir(path)] + datedirs = [d for d in subdirs if is_date_dir(d, date_fmt)] + return sorted(datedirs, key=parse_func) + + +def upload_dirs_until_free(path, target_free_mbs, dryrun): + starting_freespace = float(freespace(path)) + dirs_uploaded = 0 + files_uploaded = 0 + cur_freespace = starting_freespace + reclaimed_space = 0 + error = False + directory_list = get_directory_list(path) + + if not directory_list: + print "Target directory %s has no subdirectories!" % path + sys.exit(1) + + print "Target directory: %s" % path + print "Starting freespace: %.02f MBs" % (starting_freespace / 1024 / 1024) + print "Target freespace: %.02f MBs" % target_free_mbs + print "Image subdirectories: %s" % len(directory_list) + + if dryrun: + print + print '!!! Doing dryrun -- current free space will be estimated !!!' + + print + try: + for dir_to_upload in directory_list: + if cur_freespace >= target_free_mbs * 1024 * 1024: + break + + dir_size = directory_size(dir_to_upload) + print 'Uploading %s (%.02f MBs)' % (dir_to_upload, dir_size / 1024 / 1024) + + res = s3upload.do_upload(dir_to_upload, verbose=False, dryrun=dryrun) + files_uploaded += res['files_uploaded'] + print "%s files uploaded in %.02fs" % (res['files_uploaded'], res['sec_elapsed']) + + dirs_uploaded += 1 + reclaimed_space += dir_size + + if dryrun: + cur_freespace -= dir_size + else: + cur_freespace = float(freespace(path)) + print "%.02f MBs now free" % (cur_freespace / 1024 / 1024) + print + + except Exception: + print "An unexpected error occured!" + error = True + traceback.print_exc() + + print "---------------------------------------" + if error: + pass + else: + pass + print "Finished successfully" if not error else "!!! Terminated abnormally !!!" + print "Current free space: %.02f MBs" % (cur_freespace / 1024 / 1024) + print "Reclaimed space: %.02f MBs" % (reclaimed_space / 1024 / 1024) + print "Directories uploaded: %s" % dirs_uploaded + print "Files uploaded: %s" % files_uploaded + + +if __name__ == '__main__': + if not 4 <= len(sys.argv) <= 5: + print "usage: dailyimgupload.py workingdir path megabytes [dryrun]" + sys.exit(1) + + wd = sys.argv[1] + if not os.path.isdir(wd): + print "Invalid working directory: %s" % wd + sys.exit(1) + print "Switching working directory to %s" % wd + os.chdir(wd) + + path = sys.argv[2] + if not os.path.isdir(path): + print "invalid image directory: %s" % path + sys.exit(1) + + mbs = sys.argv[3] + try: + target_free_mbs = float(mbs) + except ValueError: + print "invalid number of megabytes: %s" % mbs + sys.exit(1) + + if len(sys.argv) == 5: + dryrun = sys.argv[4] + if dryrun in ('true', 'false'): + dryrun = dryrun == 'true' + else: + print "invalid dry run argument: %s (must be either 'true' or 'false')" % dryrun + sys.exit(1) + else: + dryrun = True + + upload_dirs_until_free(path, target_free_mbs, dryrun) -- cgit v1.2.3-70-g09d2 From 009b7730d31abc7424d23281e034780a55368944 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Sat, 6 Nov 2010 20:33:54 -0400 Subject: Fixed increment bug in dailyimgupload.py --- scripts/dailyimgupload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts/dailyimgupload.py') diff --git a/scripts/dailyimgupload.py b/scripts/dailyimgupload.py index 81ee44a..db78a83 100644 --- a/scripts/dailyimgupload.py +++ b/scripts/dailyimgupload.py @@ -92,7 +92,7 @@ def upload_dirs_until_free(path, target_free_mbs, dryrun): reclaimed_space += dir_size if dryrun: - cur_freespace -= dir_size + cur_freespace += dir_size else: cur_freespace = float(freespace(path)) print "%.02f MBs now free" % (cur_freespace / 1024 / 1024) -- cgit v1.2.3-70-g09d2 From 83845fccdfbde89f07bc670288cce2cf9f60d3b6 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Sat, 6 Nov 2010 20:39:24 -0400 Subject: dailyimgupload.py now deletes, s3upload.py recognizes keyboardinterrupt --- scripts/dailyimgupload.py | 5 +++++ scripts/s3upload.py | 2 ++ 2 files changed, 7 insertions(+) (limited to 'scripts/dailyimgupload.py') diff --git a/scripts/dailyimgupload.py b/scripts/dailyimgupload.py index db78a83..4bf5f30 100644 --- a/scripts/dailyimgupload.py +++ b/scripts/dailyimgupload.py @@ -2,6 +2,7 @@ import ctypes import datetime import os import platform +import shutil import sys import traceback import s3upload @@ -90,6 +91,10 @@ def upload_dirs_until_free(path, target_free_mbs, dryrun): dirs_uploaded += 1 reclaimed_space += dir_size + + if not dryrun: + print "Deleting %s" % dir_to_upload + shutil.rmtree(dir_to_upload) if dryrun: cur_freespace += dir_size diff --git a/scripts/s3upload.py b/scripts/s3upload.py index 163f3b6..e761ea5 100644 --- a/scripts/s3upload.py +++ b/scripts/s3upload.py @@ -22,6 +22,8 @@ def get_or_initialize_aws_connection(): def retry_func(f, count): try: f() + except KeyboardInterrupt: + raise except: if count <= 1: raise else: -- cgit v1.2.3-70-g09d2 From 2880c132823d7933a32b36c2dd5f251c43bc4856 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Sat, 6 Nov 2010 21:17:41 -0400 Subject: dailyimgupload: switched file size ints to floats --- scripts/dailyimgupload.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'scripts/dailyimgupload.py') diff --git a/scripts/dailyimgupload.py b/scripts/dailyimgupload.py index 4bf5f30..186614d 100644 --- a/scripts/dailyimgupload.py +++ b/scripts/dailyimgupload.py @@ -26,7 +26,7 @@ def directory_size(path): """ FROM: http://stackoverflow.com/questions/1392413/calculating-a-directory-size-using-python """ - total_size = 0 + total_size = 0.0 for dirpath, dirnames, filenames in os.walk(path): for f in filenames: fp = os.path.join(dirpath, f) @@ -59,7 +59,7 @@ def upload_dirs_until_free(path, target_free_mbs, dryrun): dirs_uploaded = 0 files_uploaded = 0 cur_freespace = starting_freespace - reclaimed_space = 0 + reclaimed_space = 0.0 error = False directory_list = get_directory_list(path) -- cgit v1.2.3-70-g09d2