import os from Param import Param from config import * import urllib, urllib2 from subprocess import Popen, PIPE import sys Request = urllib2.Request urlencode = urllib.urlencode urlopen = urllib2.urlopen Request = urllib2.Request urlencode = urllib.urlencode urlopen = urllib2.urlopen import sys; class ParamImg_url(Param): def __init__(self, value, key="", classname=""): super(ParamImg_url, self).__init__(classname=classname) if value: try: self.filename = self._filename_temporary(key) self.path = os.path.join(self._working_dir, self.filename) self._image_download(value, self.path) self.mimetype = self._image_mimetype(self.path) self.url = value except Exception as e: self.err_warn("Unable to download image: %s" % str(value)) self.err_warn(str(e)) def _filename_temporary(self, s): return "_tmp-{}-{}_{}".format(self._classname, self._now, s) def __dict__(self): return { 'filename' : self.filename, 'path': self.path, 'url': self.url, 'mimetype': self.mimetype } def __getitem__(self, item): return self.__dict__().__getitem__(item) def __str__(self): return str(self.__dict__()) def __nonzero__(self): return True if self.path and self.mimetype else False def _image_download(self, url, path): max_size = MAX_SIZE if self.username in SPECIAL_DOWNLOADERS: max_size = SPECIAL_DOWNLOADERS_MAX_SIZE try: self._download(url, path, max_size=max_size) except Exception as e: self.err_warn("Download failed"); def _browser_request (self, url, data=None): headers = { 'User-Agent': 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)', 'Accept': '*/*', } try: req = Request(url, data, headers) response = urlopen(req) except IOError as e: if hasattr(e, 'code'): sys.stderr.write( 'browser request error: %s - ERROR %s' % (url, e.code) ) raise IOError return response def _download(self, url, destination, max_size=MAX_SIZE): response = self._browser_request(url, None) rawimg = response.read() if len(rawimg) == 0: self.err_warn("got zero-length file") if len(rawimg) > max_size: self.err_warn("file too big: max size {} KB / {} is {} KB".format( str(MAX_SIZE/1024), destination, str(len(rawimg)/1024) ) ) f = open(destination, "w") f.write(rawimg) f.close() def _image_mimetype(self, f): try: mimetype = Popen( [BIN_IDENTIFY, f], stdout=PIPE ).communicate()[0].split(" ")[1].lower() return mimetype except Exception as e: sys.stderr.write("couldn't determine mimetype\n") raise e;