timb commiting prod changes...

1 files changed, 35 insertions, 55 deletions

diff --git a/scripts/s3upload.py b/scripts/s3upload.py
index f4a5a77..a31874b 100644
--- a/scripts/s3upload.py
+++ b/scripts/s3upload.py
@@ -9,9 +9,7 @@ CONN = None
 AWS_ACCESS_KEY_ID = 'AKIAJAQK4CDDP6I6SNVA'
 AWS_SECRET_ACCESS_KEY = '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'
 BUCKET_NAME = 'dumpfm'
-
-def parse_date(date_string, fmt='%Y%m%d'):
-    return datetime.datetime.strptime(date_string, fmt)
+COUNTER = 0
 
 def retry_func(f, count):
     try:
@@ -22,75 +20,57 @@ def retry_func(f, count):
             print 'Error! retrying %s more time(s)' % (count - 1)
             retry_func(f, count - 1)
 
-def upload_file(path, dry_run=True):
+def upload_file(path):
+    global COUNTER
     path = os.path.normpath(path)
     if path == '.' or not os.path.isfile(path):
         return
     filedata = open(path, 'rb').read()
+    size = os.path.getsize(path)
     content_type = mimetypes.guess_type(path)[0]
     if not content_type:
         content_type = 'text/plain'
 
     path = path.replace('\\', '/') # Windows hack
-    if not dry_run:
-        start = time.time()
-        def do_upload():
-            CONN.put(BUCKET_NAME, path, S3.S3Object(filedata),
-                     {'x-amz-acl': 'public-read', 'Content-Type': content_type})
-        retry_func(do_upload, 3)
-        ms_took = (time.time() - start) * 1000
-        print "uploaded %s (%0.0fms)" % (path, ms_took)
+    start = time.time()
+    def do_upload():
+        CONN.put(BUCKET_NAME, path, S3.S3Object(filedata),
+                 {'x-amz-acl': 'public-read', 'Content-Type': content_type})
+    retry_func(do_upload, 3)
+    
+    ms_took = (time.time() - start) * 1000
+    print "uploaded %s   (%0.0fms)   (%sKB)" % (path, ms_took, size / 1024)
+    COUNTER += 1
 
+def upload_directory(path):
+    for f in sorted(os.listdir(path)):
+        subpath = os.path.join(path, f)
+        if os.path.isdir(subpath):
+            upload_directory(subpath)
+        else:
+            upload_file(subpath)
 
-def do_upload(directory, start_date, end_date, dry_run=True):
+def do_upload(path):
     global CONN
     CONN = S3.AWSAuthConnection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
 
-    for subdir in sorted(os.listdir(directory)):
-        subdir_date = None
-        try:
-            subdir_date = parse_date(subdir)
-        except:
-            continue
+    start = time.time()
+
+    if os.path.isdir(path):
+        upload_directory(path)
+    else:
+        upload_file(path)
 
-        
-        if start_date <= subdir_date <= end_date:
-            counter = 0
-            print "uploading contents of %s" % subdir, 
-            for filename in os.listdir(os.path.join(directory, subdir)):
-                path = os.path.join(directory, subdir, filename)
-                upload_file(path, dry_run=dry_run)
-                counter += 1
+    s_took = (time.time() - start)
+    print "uploaded %s files in %0.0fs" % (COUNTER, s_took)
 
-            print 'handled %s files' % counter
 
 if __name__ == "__main__":
-    if not 4 <= len(sys.argv) <= 5:
-        print 'usage: s3upload.py directory startdate enddate [dryrun=true]'
+    if len(sys.argv) == 1:
+        print 'usage: s3upload.py path'
         sys.exit(1)
     
-    
-    directory = sys.argv[1]
-    start_date = sys.argv[2]
-    end_date = sys.argv[3]
-    dry_run = sys.argv[4] if len(sys.argv) == 5 else 'true'
-
-    if dry_run.lower() == 'true':
-        print 'doing dry run'
-        dry_run = True
-    else:
-        dry_run = False
-
-    try:
-        start_date = parse_date(start_date)
-    except:
-        print "invalid start date: %s" % start_date
-        sys.exit(1)
-
-    try:
-        end_date = parse_date(end_date)
-    except:
-        print "invalid end date: %s" % end_date
-        sys.exit(1)
-
-    do_upload(directory, start_date, end_date, dry_run)
+    args = sys.argv[1:]
+    for path in args:
+        do_upload(path)
+        print