1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
|
import os
from Param import Param
from Config import *
import urllib, urllib2
from subprocess import Popen, PIPE
import sys
Request = urllib2.Request
urlencode = urllib.urlencode
urlopen = urllib2.urlopen
Request = urllib2.Request
urlencode = urllib.urlencode
urlopen = urllib2.urlopen
import sys;
class ParamImg_url(Param):
def __init__(self, value, key="", classname=""):
super(ParamImg_url, self).__init__(classname=classname)
if value:
try:
self.filename = self._filename_temporary(key)
self.path = os.path.join(self._working_dir, self.filename)
self._image_download(value, self.path)
self.mimetype = self._image_mimetype(self.path)
self.url = value
except Exception as e:
self.err_warn("Unable to download image: %s" % str(value))
self.err_warn(str(e))
def _filename_temporary(self, s):
return "_tmp-{}-{}_{}".format(self._classname, self._now, s)
def __dict__(self):
return {
'filename' : self.filename,
'path': self.path,
'url': self.url,
'mimetype': self.mimetype
}
def __getitem__(self, item):
return self.__dict__().__getitem__(item)
def __str__(self):
return str(self.__dict__())
def __nonzero__(self):
return True if self.path and self.mimetype else False
def _image_download(self, url, path):
max_size = MAX_SIZE
if self.username in SPECIAL_DOWNLOADERS:
max_size = SPECIAL_DOWNLOADERS_MAX_SIZE
try:
self._download(url, path, max_size=max_size)
except Exception as e:
self.err_warn("Download failed");
def _browser_request (self, url, data=None):
headers = {
'User-Agent': 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)',
'Accept': '*/*',
}
try:
req = Request(url, data, headers)
response = urlopen(req)
except IOError as e:
if hasattr(e, 'code'):
sys.stderr.write( 'browser request error: %s - ERROR %s' % (url, e.code) )
raise IOError
return response
def _download(self, url, destination, max_size=MAX_SIZE):
response = self._browser_request(url, None)
rawimg = response.read()
if len(rawimg) == 0:
self.err_warn("got zero-length file")
if len(rawimg) > max_size:
self.err_warn("file too big: max size {} KB / {} is {} KB".format(
str(MAX_SIZE/1024),
destination,
str(len(rawimg)/1024)
)
)
f = open(destination, "w")
f.write(rawimg)
f.close()
def _image_mimetype(self, f):
try:
mimetype = Popen(
[BIN_IDENTIFY, f], stdout=PIPE
).communicate()[0].split(" ")[1].lower()
return mimetype
except Exception as e:
sys.stderr.write("couldn't determine mimetype\n")
raise e;
|
