import datetime import mimetypes import os import sys import time import S3 CONN = None AWS_ACCESS_KEY_ID = 'AKIAIOP42NFKLLJXEGJQ' AWS_SECRET_ACCESS_KEY = '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' BUCKET_NAME = 'dumpfm' def get_or_initialize_aws_connection(): global CONN if not CONN: print "Initializing AWS connection with ID %s, bucket %s" % (AWS_ACCESS_KEY_ID, BUCKET_NAME) CONN = S3.AWSAuthConnection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) return CONN def retry_func(f, count): try: f() except KeyboardInterrupt: raise except: if count <= 1: raise else: print 'Error! retrying %s more time(s)' % (count - 1) retry_func(f, count - 1) def upload_file(path, verbose=True, dryrun=False): path = os.path.normpath(path) if path == '.' or not os.path.isfile(path): return filedata = open(path, 'rb').read() size = os.path.getsize(path) content_type = mimetypes.guess_type(path)[0] if not content_type: content_type = 'text/plain' path = path.replace('\\', '/') # Windows hack start = time.time() def do_upload(): conn = get_or_initialize_aws_connection() conn.put(BUCKET_NAME, path, S3.S3Object(filedata), {'x-amz-acl': 'public-read', 'Content-Type': content_type}) if not dryrun: retry_func(do_upload, 3) ms_took = (time.time() - start) * 1000 if verbose: print "uploaded %s (%0.0fms) (%sKB)" % (path, ms_took, size / 1024) return 1 def upload_directory(path, verbose=True, dryrun=False): counter = 0 for f in sorted(os.listdir(path)): subpath = os.path.join(path, f) if os.path.isdir(subpath): counter += upload_directory(subpath, verbose=verbose, dryrun=dryrun) else: counter += upload_file(subpath, verbose=verbose, dryrun=dryrun) return counter def do_upload(path, verbose=True, dryrun=False): counter = 0 start = time.time() if os.path.isdir(path): counter += upload_directory(path, verbose=verbose, dryrun=dryrun) else: counter += upload_file(path, verbose=verbose, dryrun=dryrun) s_took = (time.time() - start) if verbose: print "uploaded %s files in %0.0fs" % (counter, s_took) return { 'sec_elapsed': s_took, 'files_uploaded': counter } if __name__ == "__main__": if len(sys.argv) == 1: print 'usage: s3upload.py path' sys.exit(1) args = sys.argv[1:] for path in args: do_upload(path) print