1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
|
import ctypes
import datetime
import os
import platform
import shutil
import sys
import traceback
import s3upload
def freespace(p):
"""
FROM: http://atlee.ca/blog/2008/02/23/getting-free-diskspace-in-python/
http://stackoverflow.com/questions/51658/cross-platform-space-remaining-on-volume-using-python
Returns the number of free bytes on the drive that ``p`` is on
"""
if platform.system() == 'Windows':
free_bytes = ctypes.c_ulonglong(0)
ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(p), None, None, ctypes.pointer(free_bytes))
return free_bytes.value
else:
s = os.statvfs(p)
return s.f_bsize * s.f_bavail
def directory_size(path):
"""
FROM: http://stackoverflow.com/questions/1392413/calculating-a-directory-size-using-python
"""
total_size = 0
for dirpath, dirnames, filenames in os.walk(path):
for f in filenames:
fp = os.path.join(dirpath, f)
total_size += os.path.getsize(fp)
return total_size
def parse_date_dir(d, date_fmt):
if not os.path.exists(d):
raise ValueError('%s does not exist' % d)
if not os.path.isdir(d):
raise ValueError('%s is not a directory' % d)
return datetime.datetime.strptime(os.path.basename(d), date_fmt)
def is_date_dir(d, date_fmt):
try:
parse_date_dir(d, date_fmt)
return True
except ValueError:
return False
def get_directory_list(path, date_fmt='%Y%m%d'):
parse_func = lambda d: parse_date_dir(d, date_fmt)
subdirs = [os.path.join(path, child) for child in os.listdir(path)]
datedirs = [d for d in subdirs if is_date_dir(d, date_fmt)]
return sorted(datedirs, key=parse_func)
def upload_dirs_until_free(path, target_free_mbs, dryrun):
starting_freespace = float(freespace(path))
dirs_uploaded = 0
files_uploaded = 0
cur_freespace = starting_freespace
reclaimed_space = 0
error = False
directory_list = get_directory_list(path)
if not directory_list:
print "Target directory %s has no subdirectories!" % path
sys.exit(1)
print "Target directory: %s" % path
print "Starting freespace: %.02f MBs" % (starting_freespace / 1024 / 1024)
print "Target freespace: %.02f MBs" % target_free_mbs
print "Image subdirectories: %s" % len(directory_list)
if dryrun:
print
print '!!! Doing dryrun -- current free space will be estimated !!!'
print
try:
for dir_to_upload in directory_list:
if cur_freespace >= target_free_mbs * 1024 * 1024:
break
dir_size = directory_size(dir_to_upload)
print 'Uploading %s (%.02f MBs)' % (dir_to_upload, dir_size / 1024 / 1024)
res = s3upload.do_upload(dir_to_upload, verbose=False, dryrun=dryrun)
files_uploaded += res['files_uploaded']
print "%s files uploaded in %.02fs" % (res['files_uploaded'], res['sec_elapsed'])
dirs_uploaded += 1
reclaimed_space += dir_size
if not dryrun:
print "Deleting %s" % dir_to_upload
shutil.rmtree(dir_to_upload)
if dryrun:
cur_freespace += dir_size
else:
cur_freespace = float(freespace(path))
print "%.02f MBs now free" % (cur_freespace / 1024 / 1024)
print
except Exception:
print "An unexpected error occured!"
error = True
traceback.print_exc()
print "---------------------------------------"
if error:
pass
else:
pass
print "Finished successfully" if not error else "!!! Terminated abnormally !!!"
print "Current free space: %.02f MBs" % (cur_freespace / 1024 / 1024)
print "Reclaimed space: %.02f MBs" % (reclaimed_space / 1024 / 1024)
print "Directories uploaded: %s" % dirs_uploaded
print "Files uploaded: %s" % files_uploaded
if __name__ == '__main__':
if not 4 <= len(sys.argv) <= 5:
print "usage: dailyimgupload.py workingdir path megabytes [dryrun]"
sys.exit(1)
wd = sys.argv[1]
if not os.path.isdir(wd):
print "Invalid working directory: %s" % wd
sys.exit(1)
print "Switching working directory to %s" % wd
os.chdir(wd)
path = sys.argv[2]
if not os.path.isdir(path):
print "invalid image directory: %s" % path
sys.exit(1)
mbs = sys.argv[3]
try:
target_free_mbs = float(mbs)
except ValueError:
print "invalid number of megabytes: %s" % mbs
sys.exit(1)
if len(sys.argv) == 5:
dryrun = sys.argv[4]
if dryrun in ('true', 'false'):
dryrun = dryrun == 'true'
else:
print "invalid dry run argument: %s (must be either 'true' or 'false')" % dryrun
sys.exit(1)
else:
dryrun = True
upload_dirs_until_free(path, target_free_mbs, dryrun)
|
