summaryrefslogtreecommitdiff
path: root/scripts
diff options
context:
space:
mode:
Diffstat (limited to 'scripts')
-rw-r--r--scripts/dailyimgupload.py314
1 files changed, 157 insertions, 157 deletions
diff --git a/scripts/dailyimgupload.py b/scripts/dailyimgupload.py
index 186614d..66f2b7b 100644
--- a/scripts/dailyimgupload.py
+++ b/scripts/dailyimgupload.py
@@ -1,157 +1,157 @@
-import ctypes
-import datetime
-import os
-import platform
-import shutil
-import sys
-import traceback
-import s3upload
-
-
-def freespace(p):
- """
- FROM: http://atlee.ca/blog/2008/02/23/getting-free-diskspace-in-python/
- http://stackoverflow.com/questions/51658/cross-platform-space-remaining-on-volume-using-python
- Returns the number of free bytes on the drive that ``p`` is on
- """
- if platform.system() == 'Windows':
- free_bytes = ctypes.c_ulonglong(0)
- ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(p), None, None, ctypes.pointer(free_bytes))
- return free_bytes.value
- else:
- s = os.statvfs(p)
- return s.f_bsize * s.f_bavail
-
-def directory_size(path):
- """
- FROM: http://stackoverflow.com/questions/1392413/calculating-a-directory-size-using-python
- """
- total_size = 0.0
- for dirpath, dirnames, filenames in os.walk(path):
- for f in filenames:
- fp = os.path.join(dirpath, f)
- total_size += os.path.getsize(fp)
- return total_size
-
-def parse_date_dir(d, date_fmt):
- if not os.path.exists(d):
- raise ValueError('%s does not exist' % d)
- if not os.path.isdir(d):
- raise ValueError('%s is not a directory' % d)
- return datetime.datetime.strptime(os.path.basename(d), date_fmt)
-
-def is_date_dir(d, date_fmt):
- try:
- parse_date_dir(d, date_fmt)
- return True
- except ValueError:
- return False
-
-def get_directory_list(path, date_fmt='%Y%m%d'):
- parse_func = lambda d: parse_date_dir(d, date_fmt)
- subdirs = [os.path.join(path, child) for child in os.listdir(path)]
- datedirs = [d for d in subdirs if is_date_dir(d, date_fmt)]
- return sorted(datedirs, key=parse_func)
-
-
-def upload_dirs_until_free(path, target_free_mbs, dryrun):
- starting_freespace = float(freespace(path))
- dirs_uploaded = 0
- files_uploaded = 0
- cur_freespace = starting_freespace
- reclaimed_space = 0.0
- error = False
- directory_list = get_directory_list(path)
-
- if not directory_list:
- print "Target directory %s has no subdirectories!" % path
- sys.exit(1)
-
- print "Target directory: %s" % path
- print "Starting freespace: %.02f MBs" % (starting_freespace / 1024 / 1024)
- print "Target freespace: %.02f MBs" % target_free_mbs
- print "Image subdirectories: %s" % len(directory_list)
-
- if dryrun:
- print
- print '!!! Doing dryrun -- current free space will be estimated !!!'
-
- print
- try:
- for dir_to_upload in directory_list:
- if cur_freespace >= target_free_mbs * 1024 * 1024:
- break
-
- dir_size = directory_size(dir_to_upload)
- print 'Uploading %s (%.02f MBs)' % (dir_to_upload, dir_size / 1024 / 1024)
-
- res = s3upload.do_upload(dir_to_upload, verbose=False, dryrun=dryrun)
- files_uploaded += res['files_uploaded']
- print "%s files uploaded in %.02fs" % (res['files_uploaded'], res['sec_elapsed'])
-
- dirs_uploaded += 1
- reclaimed_space += dir_size
-
- if not dryrun:
- print "Deleting %s" % dir_to_upload
- shutil.rmtree(dir_to_upload)
-
- if dryrun:
- cur_freespace += dir_size
- else:
- cur_freespace = float(freespace(path))
- print "%.02f MBs now free" % (cur_freespace / 1024 / 1024)
- print
-
- except Exception:
- print "An unexpected error occured!"
- error = True
- traceback.print_exc()
-
- print "---------------------------------------"
- if error:
- pass
- else:
- pass
- print "Finished successfully" if not error else "!!! Terminated abnormally !!!"
- print "Current free space: %.02f MBs" % (cur_freespace / 1024 / 1024)
- print "Reclaimed space: %.02f MBs" % (reclaimed_space / 1024 / 1024)
- print "Directories uploaded: %s" % dirs_uploaded
- print "Files uploaded: %s" % files_uploaded
-
-
-if __name__ == '__main__':
- if not 4 <= len(sys.argv) <= 5:
- print "usage: dailyimgupload.py workingdir path megabytes [dryrun]"
- sys.exit(1)
-
- wd = sys.argv[1]
- if not os.path.isdir(wd):
- print "Invalid working directory: %s" % wd
- sys.exit(1)
- print "Switching working directory to %s" % wd
- os.chdir(wd)
-
- path = sys.argv[2]
- if not os.path.isdir(path):
- print "invalid image directory: %s" % path
- sys.exit(1)
-
- mbs = sys.argv[3]
- try:
- target_free_mbs = float(mbs)
- except ValueError:
- print "invalid number of megabytes: %s" % mbs
- sys.exit(1)
-
- if len(sys.argv) == 5:
- dryrun = sys.argv[4]
- if dryrun in ('true', 'false'):
- dryrun = dryrun == 'true'
- else:
- print "invalid dry run argument: %s (must be either 'true' or 'false')" % dryrun
- sys.exit(1)
- else:
- dryrun = True
-
- upload_dirs_until_free(path, target_free_mbs, dryrun)
+import ctypes
+import datetime
+import os
+import platform
+import shutil
+import sys
+import traceback
+import s3upload
+
+
+def freespace(p):
+ """
+ FROM: http://atlee.ca/blog/2008/02/23/getting-free-diskspace-in-python/
+ http://stackoverflow.com/questions/51658/cross-platform-space-remaining-on-volume-using-python
+ Returns the number of free bytes on the drive that ``p`` is on
+ """
+ if platform.system() == 'Windows':
+ free_bytes = ctypes.c_ulonglong(0)
+ ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(p), None, None, ctypes.pointer(free_bytes))
+ return free_bytes.value
+ else:
+ s = os.statvfs(p)
+ return s.f_bsize * s.f_bavail
+
+def directory_size(path):
+ """
+ FROM: http://stackoverflow.com/questions/1392413/calculating-a-directory-size-using-python
+ """
+ total_size = 0.0
+ for dirpath, dirnames, filenames in os.walk(path):
+ for f in filenames:
+ fp = os.path.join(dirpath, f)
+ total_size += os.path.getsize(fp)
+ return total_size
+
+def parse_date_dir(d, date_fmt):
+ if not os.path.exists(d):
+ raise ValueError('%s does not exist' % d)
+ if not os.path.isdir(d):
+ raise ValueError('%s is not a directory' % d)
+ return datetime.datetime.strptime(os.path.basename(d), date_fmt)
+
+def is_date_dir(d, date_fmt):
+ try:
+ parse_date_dir(d, date_fmt)
+ return True
+ except ValueError:
+ return False
+
+def get_directory_list(path, date_fmt='%Y%m%d'):
+ parse_func = lambda d: parse_date_dir(d, date_fmt)
+ subdirs = [os.path.join(path, child) for child in os.listdir(path)]
+ datedirs = [d for d in subdirs if is_date_dir(d, date_fmt)]
+ return sorted(datedirs, key=parse_func)
+
+
+def upload_dirs_until_free(path, target_free_mbs, dryrun):
+ starting_freespace = float(freespace(path))
+ dirs_uploaded = 0
+ files_uploaded = 0
+ cur_freespace = starting_freespace
+ reclaimed_space = 0.0
+ error = False
+ directory_list = get_directory_list(path)
+
+ if not directory_list:
+ print "Target directory %s has no subdirectories!" % path
+ sys.exit(1)
+
+ print "Target directory: %s" % path
+ print "Starting freespace: %.02f MBs" % (starting_freespace / 1024 / 1024)
+ print "Target freespace: %.02f MBs" % target_free_mbs
+ print "Image subdirectories: %s" % len(directory_list)
+
+ if dryrun:
+ print
+ print '!!! Doing dryrun -- current free space will be estimated !!!'
+
+ print
+ try:
+ for dir_to_upload in directory_list:
+ if cur_freespace >= target_free_mbs * 1024 * 1024:
+ break
+
+ dir_size = directory_size(dir_to_upload)
+ print 'Uploading %s (%.02f MBs)' % (dir_to_upload, dir_size / 1024 / 1024)
+
+ res = s3upload.do_upload(dir_to_upload, verbose=False, dryrun=dryrun)
+ files_uploaded += res['files_uploaded']
+ print "%s files uploaded in %.02fs" % (res['files_uploaded'], res['sec_elapsed'])
+
+ dirs_uploaded += 1
+ reclaimed_space += dir_size
+
+ if not dryrun:
+ print "Deleting %s" % dir_to_upload
+ shutil.rmtree(dir_to_upload)
+
+ if dryrun:
+ cur_freespace += dir_size
+ else:
+ cur_freespace = float(freespace(path))
+ print "%.02f MBs now free" % (cur_freespace / 1024 / 1024)
+ print
+
+ except Exception:
+ print "An unexpected error occured!"
+ error = True
+ traceback.print_exc()
+
+ print "---------------------------------------"
+ if error:
+ pass
+ else:
+ pass
+ print "Finished successfully" if not error else "!!! Terminated abnormally !!!"
+ print "Current free space: %.02f MBs" % (cur_freespace / 1024 / 1024)
+ print "Reclaimed space: %.02f MBs" % (reclaimed_space / 1024 / 1024)
+ print "Directories uploaded: %s" % dirs_uploaded
+ print "Files uploaded: %s" % files_uploaded
+
+
+if __name__ == '__main__':
+ if not 4 <= len(sys.argv) <= 5:
+ print "usage: dailyimgupload.py workingdir path megabytes [dryrun]"
+ sys.exit(1)
+
+ wd = sys.argv[1]
+ if not os.path.isdir(wd):
+ print "Invalid working directory: %s" % wd
+ sys.exit(1)
+ print "Switching working directory to %s" % wd
+ os.chdir(wd)
+
+ path = sys.argv[2]
+ if not os.path.isdir(path):
+ print "invalid image directory: %s" % path
+ sys.exit(1)
+
+ mbs = sys.argv[3]
+ try:
+ target_free_mbs = float(mbs)
+ except ValueError:
+ print "invalid number of megabytes: %s" % mbs
+ sys.exit(1)
+
+ if len(sys.argv) == 5:
+ dryrun = sys.argv[4]
+ if dryrun in ('true', 'false'):
+ dryrun = dryrun == 'true'
+ else:
+ print "invalid dry run argument: %s (must be either 'true' or 'false')" % dryrun
+ sys.exit(1)
+ else:
+ dryrun = True
+
+ upload_dirs_until_free(path, target_free_mbs, dryrun)
generated by cgit v1.2.3-70-g09d2 (git 2.51.0) at 2025-12-10 13:00:18 +0000