From f9440deeb6dac3310efa8b1c9322885eea5b4ddf Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Sun, 7 Nov 2010 14:26:44 -0500 Subject: Fix line ending for dailyimgupload.py --- scripts/dailyimgupload.py | 314 +++++++++++++++++++++++----------------------- 1 file changed, 157 insertions(+), 157 deletions(-) (limited to 'scripts') diff --git a/scripts/dailyimgupload.py b/scripts/dailyimgupload.py index 186614d..66f2b7b 100644 --- a/scripts/dailyimgupload.py +++ b/scripts/dailyimgupload.py @@ -1,157 +1,157 @@ -import ctypes -import datetime -import os -import platform -import shutil -import sys -import traceback -import s3upload - - -def freespace(p): - """ - FROM: http://atlee.ca/blog/2008/02/23/getting-free-diskspace-in-python/ - http://stackoverflow.com/questions/51658/cross-platform-space-remaining-on-volume-using-python - Returns the number of free bytes on the drive that ``p`` is on - """ - if platform.system() == 'Windows': - free_bytes = ctypes.c_ulonglong(0) - ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(p), None, None, ctypes.pointer(free_bytes)) - return free_bytes.value - else: - s = os.statvfs(p) - return s.f_bsize * s.f_bavail - -def directory_size(path): - """ - FROM: http://stackoverflow.com/questions/1392413/calculating-a-directory-size-using-python - """ - total_size = 0.0 - for dirpath, dirnames, filenames in os.walk(path): - for f in filenames: - fp = os.path.join(dirpath, f) - total_size += os.path.getsize(fp) - return total_size - -def parse_date_dir(d, date_fmt): - if not os.path.exists(d): - raise ValueError('%s does not exist' % d) - if not os.path.isdir(d): - raise ValueError('%s is not a directory' % d) - return datetime.datetime.strptime(os.path.basename(d), date_fmt) - -def is_date_dir(d, date_fmt): - try: - parse_date_dir(d, date_fmt) - return True - except ValueError: - return False - -def get_directory_list(path, date_fmt='%Y%m%d'): - parse_func = lambda d: parse_date_dir(d, date_fmt) - subdirs = [os.path.join(path, child) for child in os.listdir(path)] - datedirs = [d for d in subdirs if is_date_dir(d, date_fmt)] - return sorted(datedirs, key=parse_func) - - -def upload_dirs_until_free(path, target_free_mbs, dryrun): - starting_freespace = float(freespace(path)) - dirs_uploaded = 0 - files_uploaded = 0 - cur_freespace = starting_freespace - reclaimed_space = 0.0 - error = False - directory_list = get_directory_list(path) - - if not directory_list: - print "Target directory %s has no subdirectories!" % path - sys.exit(1) - - print "Target directory: %s" % path - print "Starting freespace: %.02f MBs" % (starting_freespace / 1024 / 1024) - print "Target freespace: %.02f MBs" % target_free_mbs - print "Image subdirectories: %s" % len(directory_list) - - if dryrun: - print - print '!!! Doing dryrun -- current free space will be estimated !!!' - - print - try: - for dir_to_upload in directory_list: - if cur_freespace >= target_free_mbs * 1024 * 1024: - break - - dir_size = directory_size(dir_to_upload) - print 'Uploading %s (%.02f MBs)' % (dir_to_upload, dir_size / 1024 / 1024) - - res = s3upload.do_upload(dir_to_upload, verbose=False, dryrun=dryrun) - files_uploaded += res['files_uploaded'] - print "%s files uploaded in %.02fs" % (res['files_uploaded'], res['sec_elapsed']) - - dirs_uploaded += 1 - reclaimed_space += dir_size - - if not dryrun: - print "Deleting %s" % dir_to_upload - shutil.rmtree(dir_to_upload) - - if dryrun: - cur_freespace += dir_size - else: - cur_freespace = float(freespace(path)) - print "%.02f MBs now free" % (cur_freespace / 1024 / 1024) - print - - except Exception: - print "An unexpected error occured!" - error = True - traceback.print_exc() - - print "---------------------------------------" - if error: - pass - else: - pass - print "Finished successfully" if not error else "!!! Terminated abnormally !!!" - print "Current free space: %.02f MBs" % (cur_freespace / 1024 / 1024) - print "Reclaimed space: %.02f MBs" % (reclaimed_space / 1024 / 1024) - print "Directories uploaded: %s" % dirs_uploaded - print "Files uploaded: %s" % files_uploaded - - -if __name__ == '__main__': - if not 4 <= len(sys.argv) <= 5: - print "usage: dailyimgupload.py workingdir path megabytes [dryrun]" - sys.exit(1) - - wd = sys.argv[1] - if not os.path.isdir(wd): - print "Invalid working directory: %s" % wd - sys.exit(1) - print "Switching working directory to %s" % wd - os.chdir(wd) - - path = sys.argv[2] - if not os.path.isdir(path): - print "invalid image directory: %s" % path - sys.exit(1) - - mbs = sys.argv[3] - try: - target_free_mbs = float(mbs) - except ValueError: - print "invalid number of megabytes: %s" % mbs - sys.exit(1) - - if len(sys.argv) == 5: - dryrun = sys.argv[4] - if dryrun in ('true', 'false'): - dryrun = dryrun == 'true' - else: - print "invalid dry run argument: %s (must be either 'true' or 'false')" % dryrun - sys.exit(1) - else: - dryrun = True - - upload_dirs_until_free(path, target_free_mbs, dryrun) +import ctypes +import datetime +import os +import platform +import shutil +import sys +import traceback +import s3upload + + +def freespace(p): + """ + FROM: http://atlee.ca/blog/2008/02/23/getting-free-diskspace-in-python/ + http://stackoverflow.com/questions/51658/cross-platform-space-remaining-on-volume-using-python + Returns the number of free bytes on the drive that ``p`` is on + """ + if platform.system() == 'Windows': + free_bytes = ctypes.c_ulonglong(0) + ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(p), None, None, ctypes.pointer(free_bytes)) + return free_bytes.value + else: + s = os.statvfs(p) + return s.f_bsize * s.f_bavail + +def directory_size(path): + """ + FROM: http://stackoverflow.com/questions/1392413/calculating-a-directory-size-using-python + """ + total_size = 0.0 + for dirpath, dirnames, filenames in os.walk(path): + for f in filenames: + fp = os.path.join(dirpath, f) + total_size += os.path.getsize(fp) + return total_size + +def parse_date_dir(d, date_fmt): + if not os.path.exists(d): + raise ValueError('%s does not exist' % d) + if not os.path.isdir(d): + raise ValueError('%s is not a directory' % d) + return datetime.datetime.strptime(os.path.basename(d), date_fmt) + +def is_date_dir(d, date_fmt): + try: + parse_date_dir(d, date_fmt) + return True + except ValueError: + return False + +def get_directory_list(path, date_fmt='%Y%m%d'): + parse_func = lambda d: parse_date_dir(d, date_fmt) + subdirs = [os.path.join(path, child) for child in os.listdir(path)] + datedirs = [d for d in subdirs if is_date_dir(d, date_fmt)] + return sorted(datedirs, key=parse_func) + + +def upload_dirs_until_free(path, target_free_mbs, dryrun): + starting_freespace = float(freespace(path)) + dirs_uploaded = 0 + files_uploaded = 0 + cur_freespace = starting_freespace + reclaimed_space = 0.0 + error = False + directory_list = get_directory_list(path) + + if not directory_list: + print "Target directory %s has no subdirectories!" % path + sys.exit(1) + + print "Target directory: %s" % path + print "Starting freespace: %.02f MBs" % (starting_freespace / 1024 / 1024) + print "Target freespace: %.02f MBs" % target_free_mbs + print "Image subdirectories: %s" % len(directory_list) + + if dryrun: + print + print '!!! Doing dryrun -- current free space will be estimated !!!' + + print + try: + for dir_to_upload in directory_list: + if cur_freespace >= target_free_mbs * 1024 * 1024: + break + + dir_size = directory_size(dir_to_upload) + print 'Uploading %s (%.02f MBs)' % (dir_to_upload, dir_size / 1024 / 1024) + + res = s3upload.do_upload(dir_to_upload, verbose=False, dryrun=dryrun) + files_uploaded += res['files_uploaded'] + print "%s files uploaded in %.02fs" % (res['files_uploaded'], res['sec_elapsed']) + + dirs_uploaded += 1 + reclaimed_space += dir_size + + if not dryrun: + print "Deleting %s" % dir_to_upload + shutil.rmtree(dir_to_upload) + + if dryrun: + cur_freespace += dir_size + else: + cur_freespace = float(freespace(path)) + print "%.02f MBs now free" % (cur_freespace / 1024 / 1024) + print + + except Exception: + print "An unexpected error occured!" + error = True + traceback.print_exc() + + print "---------------------------------------" + if error: + pass + else: + pass + print "Finished successfully" if not error else "!!! Terminated abnormally !!!" + print "Current free space: %.02f MBs" % (cur_freespace / 1024 / 1024) + print "Reclaimed space: %.02f MBs" % (reclaimed_space / 1024 / 1024) + print "Directories uploaded: %s" % dirs_uploaded + print "Files uploaded: %s" % files_uploaded + + +if __name__ == '__main__': + if not 4 <= len(sys.argv) <= 5: + print "usage: dailyimgupload.py workingdir path megabytes [dryrun]" + sys.exit(1) + + wd = sys.argv[1] + if not os.path.isdir(wd): + print "Invalid working directory: %s" % wd + sys.exit(1) + print "Switching working directory to %s" % wd + os.chdir(wd) + + path = sys.argv[2] + if not os.path.isdir(path): + print "invalid image directory: %s" % path + sys.exit(1) + + mbs = sys.argv[3] + try: + target_free_mbs = float(mbs) + except ValueError: + print "invalid number of megabytes: %s" % mbs + sys.exit(1) + + if len(sys.argv) == 5: + dryrun = sys.argv[4] + if dryrun in ('true', 'false'): + dryrun = dryrun == 'true' + else: + print "invalid dry run argument: %s (must be either 'true' or 'false')" % dryrun + sys.exit(1) + else: + dryrun = True + + upload_dirs_until_free(path, target_free_mbs, dryrun) -- cgit v1.2.3-70-g09d2