1 files changed, 152 insertions, 0 deletions
diff --git a/scripts/dailyimgupload.py b/scripts/dailyimgupload.py
new file mode 100644
index 0000000..81ee44a
--- /dev/null
+++ b/scripts/dailyimgupload.py
@@ -0,0 +1,152 @@
+import ctypes
+import datetime
+import os
+import platform
+import sys
+import traceback
+import s3upload
+
+
+def freespace(p):
+     """
+     FROM: http://atlee.ca/blog/2008/02/23/getting-free-diskspace-in-python/
+           http://stackoverflow.com/questions/51658/cross-platform-space-remaining-on-volume-using-python
+     Returns the number of free bytes on the drive that ``p`` is on
+     """
+     if platform.system() == 'Windows':
+         free_bytes = ctypes.c_ulonglong(0)
+         ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(p), None, None, ctypes.pointer(free_bytes))
+         return free_bytes.value
+     else:
+         s = os.statvfs(p)
+         return s.f_bsize * s.f_bavail
+
+def directory_size(path):
+    """
+    FROM: http://stackoverflow.com/questions/1392413/calculating-a-directory-size-using-python
+    """
+    total_size = 0
+    for dirpath, dirnames, filenames in os.walk(path):
+        for f in filenames:
+            fp = os.path.join(dirpath, f)
+            total_size += os.path.getsize(fp)
+    return total_size
+
+def parse_date_dir(d, date_fmt):
+    if not os.path.exists(d):
+        raise ValueError('%s does not exist' % d)
+    if not os.path.isdir(d):
+        raise ValueError('%s is not a directory' % d)
+    return datetime.datetime.strptime(os.path.basename(d), date_fmt)
+
+def is_date_dir(d, date_fmt):
+    try:
+        parse_date_dir(d, date_fmt)
+        return True
+    except ValueError:
+        return False
+
+def get_directory_list(path, date_fmt='%Y%m%d'):
+    parse_func = lambda d: parse_date_dir(d, date_fmt)
+    subdirs = [os.path.join(path, child) for child in os.listdir(path)]
+    datedirs = [d for d in subdirs if is_date_dir(d, date_fmt)]
+    return sorted(datedirs, key=parse_func)
+    
+
+def upload_dirs_until_free(path, target_free_mbs, dryrun):
+    starting_freespace = float(freespace(path))
+    dirs_uploaded = 0
+    files_uploaded = 0
+    cur_freespace = starting_freespace
+    reclaimed_space = 0
+    error = False
+    directory_list = get_directory_list(path)
+
+    if not directory_list:
+        print "Target directory %s has no subdirectories!" % path
+        sys.exit(1)
+
+    print "Target directory: %s" % path
+    print "Starting freespace: %.02f MBs" % (starting_freespace / 1024 / 1024)
+    print "Target freespace: %.02f MBs"  % target_free_mbs
+    print "Image subdirectories: %s" % len(directory_list)
+
+    if dryrun:
+        print
+        print '!!! Doing dryrun -- current free space will be estimated !!!'
+
+    print
+    try:
+        for dir_to_upload in directory_list:
+            if cur_freespace >= target_free_mbs * 1024 * 1024:
+                break
+        
+            dir_size = directory_size(dir_to_upload)
+            print 'Uploading %s (%.02f MBs)' % (dir_to_upload, dir_size / 1024 / 1024)
+
+            res = s3upload.do_upload(dir_to_upload, verbose=False, dryrun=dryrun)
+            files_uploaded += res['files_uploaded']
+            print "%s files uploaded in %.02fs" % (res['files_uploaded'], res['sec_elapsed'])
+
+            dirs_uploaded += 1
+            reclaimed_space += dir_size
+                
+            if dryrun:
+                cur_freespace -= dir_size 
+            else:
+                cur_freespace = float(freespace(path))
+            print "%.02f MBs now free" % (cur_freespace / 1024 / 1024)
+            print
+
+    except Exception:
+        print "An unexpected error occured!"
+        error = True
+        traceback.print_exc()
+
+    print "---------------------------------------"
+    if error:
+        pass
+    else:
+        pass
+    print "Finished successfully" if not error else "!!! Terminated abnormally !!!"
+    print "Current free space: %.02f MBs" % (cur_freespace / 1024 / 1024)
+    print "Reclaimed space: %.02f MBs" % (reclaimed_space / 1024 / 1024)
+    print "Directories uploaded: %s" % dirs_uploaded
+    print "Files uploaded: %s" % files_uploaded
+
+
+if __name__ == '__main__':
+    if not 4 <= len(sys.argv) <= 5:
+        print "usage: dailyimgupload.py workingdir path megabytes [dryrun]"
+        sys.exit(1)
+
+    wd = sys.argv[1]
+    if not os.path.isdir(wd):
+        print "Invalid working directory: %s" % wd
+        sys.exit(1)
+    print "Switching working directory to %s" % wd
+    os.chdir(wd)
+
+    path = sys.argv[2]
+    if not os.path.isdir(path):
+        print "invalid image directory: %s" % path
+        sys.exit(1)
+
+    mbs = sys.argv[3]
+    try:
+        target_free_mbs = float(mbs)
+    except ValueError:
+        print "invalid number of megabytes: %s" % mbs
+        sys.exit(1)
+
+    if len(sys.argv) == 5:
+        dryrun = sys.argv[4]
+        if dryrun in ('true', 'false'):
+            dryrun = dryrun == 'true'
+        else:
+            print "invalid dry run argument: %s (must be either 'true' or 'false')" % dryrun
+            sys.exit(1)
+    else:
+        dryrun = True
+
+    upload_dirs_until_free(path, target_free_mbs, dryrun)