From d765ecafa41542f3745522c164f9c8ed9bb0eb62 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Sat, 6 Nov 2010 20:29:40 -0400 Subject: Added dailyimgupload.py, updated s3upload.py --- scripts/s3upload.py | 48 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 16 deletions(-) (limited to 'scripts/s3upload.py') diff --git a/scripts/s3upload.py b/scripts/s3upload.py index 724561c..9263a8f 100644 --- a/scripts/s3upload.py +++ b/scripts/s3upload.py @@ -9,7 +9,15 @@ CONN = None AWS_ACCESS_KEY_ID = 'AKIAIOP42NFKLLJXEGJQ' AWS_SECRET_ACCESS_KEY = '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' BUCKET_NAME = 'dumpfm' -COUNTER = 0 + +def get_or_initialize_aws_connection(): + global CONN + if not CONN: + print "Initializing AWS connection with ID %s, bucket %s" % (AWS_ACCESS_KEY_ID, + BUCKET_NAME) + CONN = S3.AWSAuthConnection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) + return CONN + def retry_func(f, count): try: @@ -20,8 +28,7 @@ def retry_func(f, count): print 'Error! retrying %s more time(s)' % (count - 1) retry_func(f, count - 1) -def upload_file(path): - global COUNTER +def upload_file(path, conn, verbose=True, dryrun=False): path = os.path.normpath(path) if path == '.' or not os.path.isfile(path): return @@ -34,35 +41,44 @@ def upload_file(path): path = path.replace('\\', '/') # Windows hack start = time.time() def do_upload(): - CONN.put(BUCKET_NAME, path, S3.S3Object(filedata), + conn.put(BUCKET_NAME, path, S3.S3Object(filedata), {'x-amz-acl': 'public-read', 'Content-Type': content_type}) - retry_func(do_upload, 3) + + if not dryrun: + retry_func(do_upload, 3) ms_took = (time.time() - start) * 1000 - print "uploaded %s (%0.0fms) (%sKB)" % (path, ms_took, size / 1024) - COUNTER += 1 + if verbose: + print "uploaded %s (%0.0fms) (%sKB)" % (path, ms_took, size / 1024) + return 1 -def upload_directory(path): +def upload_directory(path, conn, verbose=True, dryrun=False): + counter = 0 for f in sorted(os.listdir(path)): subpath = os.path.join(path, f) if os.path.isdir(subpath): - upload_directory(subpath) + counter += upload_directory(subpath, conn, verbose=verbose, dryrun=dryrun) else: - upload_file(subpath) + counter += upload_file(subpath, conn, verbose=verbose, dryrun=dryrun) + return counter -def do_upload(path): - global CONN - CONN = S3.AWSAuthConnection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) +def do_upload(path, verbose=True, dryrun=False): + conn = get_or_initialize_aws_connection() + counter = 0 start = time.time() if os.path.isdir(path): - upload_directory(path) + counter += upload_directory(path, conn, verbose=verbose, dryrun=dryrun) else: - upload_file(path) + counter += upload_file(path, conn, verbose=verbose, dryrun=dryrun) s_took = (time.time() - start) - print "uploaded %s files in %0.0fs" % (COUNTER, s_took) + + if verbose: + print "uploaded %s files in %0.0fs" % (counter, s_took) + return { 'sec_elapsed': s_took, + 'files_uploaded': counter } if __name__ == "__main__": -- cgit v1.2.3-70-g09d2 From f9b9a3b3b1d47bc69213f3e8eb88fe76691aad33 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Sat, 6 Nov 2010 20:32:36 -0400 Subject: s3upload.py now only initializes AWS connection when uploading --- scripts/s3upload.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'scripts/s3upload.py') diff --git a/scripts/s3upload.py b/scripts/s3upload.py index 9263a8f..163f3b6 100644 --- a/scripts/s3upload.py +++ b/scripts/s3upload.py @@ -28,7 +28,7 @@ def retry_func(f, count): print 'Error! retrying %s more time(s)' % (count - 1) retry_func(f, count - 1) -def upload_file(path, conn, verbose=True, dryrun=False): +def upload_file(path, verbose=True, dryrun=False): path = os.path.normpath(path) if path == '.' or not os.path.isfile(path): return @@ -41,6 +41,7 @@ def upload_file(path, conn, verbose=True, dryrun=False): path = path.replace('\\', '/') # Windows hack start = time.time() def do_upload(): + conn = get_or_initialize_aws_connection() conn.put(BUCKET_NAME, path, S3.S3Object(filedata), {'x-amz-acl': 'public-read', 'Content-Type': content_type}) @@ -52,26 +53,25 @@ def upload_file(path, conn, verbose=True, dryrun=False): print "uploaded %s (%0.0fms) (%sKB)" % (path, ms_took, size / 1024) return 1 -def upload_directory(path, conn, verbose=True, dryrun=False): +def upload_directory(path, verbose=True, dryrun=False): counter = 0 for f in sorted(os.listdir(path)): subpath = os.path.join(path, f) if os.path.isdir(subpath): - counter += upload_directory(subpath, conn, verbose=verbose, dryrun=dryrun) + counter += upload_directory(subpath, verbose=verbose, dryrun=dryrun) else: - counter += upload_file(subpath, conn, verbose=verbose, dryrun=dryrun) + counter += upload_file(subpath, verbose=verbose, dryrun=dryrun) return counter def do_upload(path, verbose=True, dryrun=False): - conn = get_or_initialize_aws_connection() counter = 0 start = time.time() if os.path.isdir(path): - counter += upload_directory(path, conn, verbose=verbose, dryrun=dryrun) + counter += upload_directory(path, verbose=verbose, dryrun=dryrun) else: - counter += upload_file(path, conn, verbose=verbose, dryrun=dryrun) + counter += upload_file(path, verbose=verbose, dryrun=dryrun) s_took = (time.time() - start) -- cgit v1.2.3-70-g09d2 From 83845fccdfbde89f07bc670288cce2cf9f60d3b6 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Sat, 6 Nov 2010 20:39:24 -0400 Subject: dailyimgupload.py now deletes, s3upload.py recognizes keyboardinterrupt --- scripts/dailyimgupload.py | 5 +++++ scripts/s3upload.py | 2 ++ 2 files changed, 7 insertions(+) (limited to 'scripts/s3upload.py') diff --git a/scripts/dailyimgupload.py b/scripts/dailyimgupload.py index db78a83..4bf5f30 100644 --- a/scripts/dailyimgupload.py +++ b/scripts/dailyimgupload.py @@ -2,6 +2,7 @@ import ctypes import datetime import os import platform +import shutil import sys import traceback import s3upload @@ -90,6 +91,10 @@ def upload_dirs_until_free(path, target_free_mbs, dryrun): dirs_uploaded += 1 reclaimed_space += dir_size + + if not dryrun: + print "Deleting %s" % dir_to_upload + shutil.rmtree(dir_to_upload) if dryrun: cur_freespace += dir_size diff --git a/scripts/s3upload.py b/scripts/s3upload.py index 163f3b6..e761ea5 100644 --- a/scripts/s3upload.py +++ b/scripts/s3upload.py @@ -22,6 +22,8 @@ def get_or_initialize_aws_connection(): def retry_func(f, count): try: f() + except KeyboardInterrupt: + raise except: if count <= 1: raise else: -- cgit v1.2.3-70-g09d2