Added dailyimgupload.py, updated s3upload.py

1 files changed, 32 insertions, 16 deletions

diff --git a/scripts/s3upload.py b/scripts/s3upload.py
index 724561c..9263a8f 100644
--- a/scripts/s3upload.py
+++ b/scripts/s3upload.py
@@ -9,7 +9,15 @@ CONN = None
 AWS_ACCESS_KEY_ID = 'AKIAIOP42NFKLLJXEGJQ'
 AWS_SECRET_ACCESS_KEY = '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'
 BUCKET_NAME = 'dumpfm'
-COUNTER = 0
+
+def get_or_initialize_aws_connection():
+    global CONN
+    if not CONN:
+        print "Initializing AWS connection with ID %s, bucket %s" % (AWS_ACCESS_KEY_ID,
+                                                                     BUCKET_NAME)
+        CONN = S3.AWSAuthConnection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
+    return CONN
+        
 
 def retry_func(f, count):
     try:
@@ -20,8 +28,7 @@ def retry_func(f, count):
             print 'Error! retrying %s more time(s)' % (count - 1)
             retry_func(f, count - 1)
 
-def upload_file(path):
-    global COUNTER
+def upload_file(path, conn, verbose=True, dryrun=False):
     path = os.path.normpath(path)
     if path == '.' or not os.path.isfile(path):
         return
@@ -34,35 +41,44 @@ def upload_file(path):
     path = path.replace('\\', '/') # Windows hack
     start = time.time()
     def do_upload():
-        CONN.put(BUCKET_NAME, path, S3.S3Object(filedata),
+        conn.put(BUCKET_NAME, path, S3.S3Object(filedata),
                  {'x-amz-acl': 'public-read', 'Content-Type': content_type})
-    retry_func(do_upload, 3)
+
+    if not dryrun:
+        retry_func(do_upload, 3)
     
     ms_took = (time.time() - start) * 1000
-    print "uploaded %s   (%0.0fms)   (%sKB)" % (path, ms_took, size / 1024)
-    COUNTER += 1
+    if verbose:
+        print "uploaded %s   (%0.0fms)   (%sKB)" % (path, ms_took, size / 1024)
+    return 1
 
-def upload_directory(path):
+def upload_directory(path, conn, verbose=True, dryrun=False):
+    counter = 0
     for f in sorted(os.listdir(path)):
         subpath = os.path.join(path, f)
         if os.path.isdir(subpath):
-            upload_directory(subpath)
+            counter += upload_directory(subpath, conn, verbose=verbose, dryrun=dryrun)
         else:
-            upload_file(subpath)
+            counter += upload_file(subpath, conn, verbose=verbose, dryrun=dryrun)
+    return counter
 
-def do_upload(path):
-    global CONN
-    CONN = S3.AWSAuthConnection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
+def do_upload(path, verbose=True, dryrun=False):
+    conn = get_or_initialize_aws_connection()
+    counter = 0
 
     start = time.time()
 
     if os.path.isdir(path):
-        upload_directory(path)
+        counter += upload_directory(path, conn, verbose=verbose, dryrun=dryrun)
     else:
-        upload_file(path)
+        counter += upload_file(path, conn, verbose=verbose, dryrun=dryrun)
 
     s_took = (time.time() - start)
-    print "uploaded %s files in %0.0fs" % (COUNTER, s_took)
+
+    if verbose:
+        print "uploaded %s files in %0.0fs" % (counter, s_took)
+    return { 'sec_elapsed': s_took,
+             'files_uploaded': counter }
 
 
 if __name__ == "__main__":