1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
|
import datetime
import mimetypes
import os
import sys
import time
import S3
CONN = None
AWS_ACCESS_KEY_ID = 'AKIAJAQK4CDDP6I6SNVA'
AWS_SECRET_ACCESS_KEY = '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'
BUCKET_NAME = 'dumpfm'
COUNTER = 0
def retry_func(f, count):
try:
f()
except:
if count <= 1: raise
else:
print 'Error! retrying %s more time(s)' % (count - 1)
retry_func(f, count - 1)
def upload_file(path):
global COUNTER
path = os.path.normpath(path)
if path == '.' or not os.path.isfile(path):
return
filedata = open(path, 'rb').read()
size = os.path.getsize(path)
content_type = mimetypes.guess_type(path)[0]
if not content_type:
content_type = 'text/plain'
path = path.replace('\\', '/') # Windows hack
start = time.time()
def do_upload():
CONN.put(BUCKET_NAME, path, S3.S3Object(filedata),
{'x-amz-acl': 'public-read', 'Content-Type': content_type})
retry_func(do_upload, 3)
ms_took = (time.time() - start) * 1000
print "uploaded %s (%0.0fms) (%sKB)" % (path, ms_took, size / 1024)
COUNTER += 1
def upload_directory(path):
for f in sorted(os.listdir(path)):
subpath = os.path.join(path, f)
if os.path.isdir(subpath):
upload_directory(subpath)
else:
upload_file(subpath)
def do_upload(path):
global CONN
CONN = S3.AWSAuthConnection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
start = time.time()
if os.path.isdir(path):
upload_directory(path)
else:
upload_file(path)
s_took = (time.time() - start)
print "uploaded %s files in %0.0fs" % (COUNTER, s_took)
if __name__ == "__main__":
if len(sys.argv) == 1:
print 'usage: s3upload.py path'
sys.exit(1)
args = sys.argv[1:]
for path in args:
do_upload(path)
print
|
