diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rwxr-xr-x | generate | 376 | ||||
| -rwxr-xr-x | pbserver.py | 26 | ||||
| -rw-r--r-- | s3.py | 2 |
4 files changed, 391 insertions, 14 deletions
@@ -3,3 +3,4 @@ *.swp *.swo *.pyc +*.un~ diff --git a/generate b/generate new file mode 100755 index 0000000..0585b10 --- /dev/null +++ b/generate @@ -0,0 +1,376 @@ +#!/usr/bin/python2.7 +import sys +import os +import re +import time +import string +import urllib, urllib2 +from subprocess import Popen, PIPE +import sha +import simplejson as json + +import mimetypes + +urlencode = urllib.urlencode +urlopen = urllib2.urlopen +Request = urllib2.Request + +MAX_SIZE = 1024 * 1024 * 1.2 * 1.5 +WORKING_DIR = "/var/www/cache" +LIKE_A_BOSS = "ryz pepper seamonkey JAMES".split(" ") + +BIN_CONVERT = "/usr/bin/convert" +BIN_COMPOSITE = "/usr/bin/composite" +BIN_IDENTIFY = "/usr/bin/identify" +DEFAULT_FINALFORMAT = "gif" +DB_TAG = "pb"; + +GRAVITY = "NorthWest North NorthEast West Center East SouthWest South SouthEast".split(" ") + +#{{{Utility functions +def is_color(s): + if s == "": + return "transparent" + if re.match('(rgba?\([0-9]+,[0-9]+,[0-9]+\))|([a-zA-Z]+)|(\#[A-Ha-h0-9]+)', s): + return s.replace(' ', ''); + else: + sys.stderr.write("Not a color: {}\n".format(s)) + raise ValueError +def bool_correct(s): + if re.match(r'^false$', s, re.IGNORECASE): + return False + elif re.match(r'^true$', s, re.IGNORECASE): + return True + else: + return s + +class dotdict(dict): + """dot.notation access to dictionary attributes""" + def __getattr__(self, attr): + return self.get(attr) + __setattr__= dict.__setitem__ + __delattr__= dict.__delitem__ + +def get_mimetype(f): + try: + mimetype = Popen( + [BIN_IDENTIFY, f], stdout=PIPE + ).communicate()[0].split(" ")[1].lower() + return mimetype + except Exception as e: + sys.stderr.write("IMGRID couldn't determine mimetype") + sys.stderr.write(str(e)) + raise; + +def sanitize (str): + return re.sub(r'\W+', '', str) + +def now(): + return int(time.time()) + +def browser_request (url, data=None): + headers = { + 'User-Agent': 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)', + 'Accept': '*/*', + } + try: + req = Request(url, data, headers) + response = urlopen(req) + except IOError, e: + if hasattr(e, 'code'): + sys.stderr.write( '%s - ERROR %s' % (url, e.code) ) + raise; + return None + else: + return response + +def download(url, destination, max_size=MAX_SIZE): + response = browser_request(url, None) + rawimg = response.read() + if len(rawimg) == 0: + sys.stderr.write("got zero-length file") + raise; + if len(rawimg) > max_size: + sys.stderr.write("file too big: max size {} KB / {} is {} KB".format( + str(MAX_SIZE/1024), + destination, + str(len(rawimg)/1024) + )) + raise; + f = open(destination, "w") + f.write(rawimg) + f.close() + +def file_size (filepath): + try: + return os.stat(file)[6] + except Exception as e: + sys.stderr.write("IMGRID couldn't determine file size") + sys.stderr.write(str(e)) + raise; + +def gif_frames(filepath): + try: + info = Popen([BIN_IDENTIFY,filepath], stdout=PIPE).communicate()[0] + frames = filter((lambda x: x), map( + (lambda x: x.split(" ")[0]), + (info).split('\n') + )) + return frames + except Exception as e: + sys.stderr.write("IMGRID couldn't get gif frames") + sys.stderr.write(str(e)) + raise; +#}}} + +class Photoblaster(): + def __init__(self, params): + self.params = {} + self.tag = "im" + self.now = now() + self.files_created = [] + self.commands = []; + self._required_keys = [ +#{{{ required_keys + "url", + "transparent", + "subtract", + "fuzz", + "width", + "height", + "black", + "white", + "brightness", + "contrast", + "saturation", + "hue", + "rotate", + "flip", + "flop", + "background", + "merge_early", + "compose", + "gravity", + "tile", + "format", + "name", + "callback", + "coalesce", + "nearest", + "dispose", +#}}} + ] + for k in self._required_keys: + if k in params: + if k in [ 'url', 'background' ] and bool_correct(params[k]): + self.params[k] = {} + self.params[k]['url'] = params[k] + self.params[k]['filename'] = "PBTMP{}_{}".format(now(), k) + + self.params[k]['path'] = os.path.join(WORKING_DIR, self.params[k]['filename']) + try: + download(self.params[k]['url'], self.params[k]['path']) + self.files_created.append(self.params[k]['path']) + self.params[k]['mimetype'] = get_mimetype(self.params[k]['path']) + frames = gif_frames(self.params[k]['path']) + if len(frames) > 1: + self.params[k]['path'] = random.choice(frames) + + + except Exception as e: + sys.stderr.write("BAD PARAMS\n") + sys.stderr.write(str(e)) + raise; + elif k in [ 'black', 'white', 'subtract' ]: + try: + self.params[k] = is_color(params[k]) + except Exception as e: + sys.stderr.write("Unable to process color for:\n") + sys.stderr.write(k) + raise e + else: + self.params[k] = bool_correct(sanitize(params[k])) + else: + self.params[k] = False; + + self.params = dotdict(self.params) + + self.basename = self._get_filename(); + + if not self.params.format: + self.params.format = DEFAULT_FINALFORMAT + self.filename = "{}.{}".format(self.basename, self.params.format) + #final filepath is stored in self.filepath + self.filepath = os.path.join(WORKING_DIR, self.filename) + + def _get_filename(self): + return "{}_{}_{}".format( + self.tag, + now(), + self.params.username or "" + ); + + def _call_cmd(self, cmd, error=""): + try: + call(cmd) + self.commands.append(" ".join(cmd)); + except Exception as e: + sys.stderr.write("IT HIT AN ERROR") + sys.stderr.write(str(cmd)) + if error: + sys.stderr.write(error) + else: + sys.stderr.write(str(e)) + + def _cleanup(self): + if not len(self.files_created): + pass + cmd = ["rm", "-f"] + self.files_created + self._call_cmd(cmd) + +def bin_composite (params, bottomdir, bottomfile, topdir, topfile, newfile): + cmd = [BIN_CONVERT] + cmd.append(os.path.join(bottomdir,bottomfile)) + cmd.append('null:') + cmd.append(os.path.join(topdir,topfile)) + cmd.append("-matte") + cmd.extend(params) + cmd.append("-layers composite") + cmd.append(os.path.join(topdir, newfile)) +# g.write("bin_composite command: %s" % (" ".join(cmd))); + os.system(" ".join(cmd)) + os.system("rm "+os.path.join(bottomdir,bottomfile)+" "+os.path.join(topdir,topfile)) + +def bin_convert (directory, params, oldfile, newfile): + cmd = [BIN_CONVERT, os.path.join(directory, oldfile)] + for x in params: + cmd.append(x) + cmd.append(os.path.join(directory,newfile)) + os.system(" ".join(cmd)) +# g.write("bin_convert command: %s" % (" ".join(cmd))); + os.system("rm "+os.path.join(directory,oldfile)) + + def _build_cmd(self): + cmd = [] + #FIXME test if number + if self.params.rotate: cmd += ["-rotate", self.params.rotate ] + if self.params.flip: cmd += ["-flip"] + if self.params.flop: cmd += ["-flop"] + +if param['transparent'] == "true": + tag = "transparent" + if is_number(param['fuzz']) and param['fuzz'] != 0: + cmd.append("-fuzz") + cmd.append(param['fuzz']+"%") + subtract_color = as_color(param['subtract'], "white") + cmd.append("-transparent") + cmd.append(subtract_color) + +VALID_DISPOSE_METHODS=["none","previous","background"] +dispose = "None" + +if param['width'] is not None and is_number(param['width']): + if int(param['width']) > 1000 and NOT_A_BOSS: + error ("width cannot be greater than 1000") + width = param['width'] +if param['height'] is not None and is_number(param['height']): + if int(param['height']) > 1000 and NOT_A_BOSS: + error ("height cannot be greater than 1000") + height = param['height'] + +if (width or height): + if param['nearest'] == "true": + if format == "gif": + cmd.append("-coalesce") + cmd.append("+map") + cmd.append("-interpolate") + cmd.append("Nearest") + cmd.append("-interpolative-resize") + else: + cmd.append("-resize") + +if width and height: + cmd.append(width + "x" + height) +elif width: + cmd.append(width) +elif height: + cmd.append("x" + height) + +if param['black'] != "black" or param['white'] != 'white': + try: + black = as_color(param['black'], "black") + white = as_color(param['white'], "white") + cmd.append("+level-colors") + cmd.append(black+","+white) + except (): + pass + +if param['contrast'] is not None and is_number(param['contrast']): + cmd.append("-contrast-stretch") + cmd.append(param['contrast']) + pass + +if param['brightness'] is not None or param['saturation'] is not None or param['hue'] is not None: + bstring = '' + if is_number(param['brightness']): + bstring += param['brightness'] + else: + bstring += "100" + bstring += ',' + if is_number(param['contrast']): + bstring += param['contrast'] + else: + bstring += "100" + bstring += ',' + if is_number(param['hue']): + bstring += param['hue'] + if bstring != "100,100,": + cmd.append("-modulate") + cmd.append(bstring) + +if bgfile is not None: + tag = param['compose'] + gravity = param['gravity'] + + if gravity not in GRAVITY: + gravity = 'center' + + + compositefile = "composite_" + newfile + compositeparams = ["-dispose", "None", "-gravity", gravity] + compositeparams.extend([ "-compose", param['compose'] ]) + + cmd.append( "-coalesce") + bin_convert (WORKING_DIR, cmd, oldfile, compositefile) + bin_composite (compositeparams, WORKING_DIR, bgfile, WORKING_DIR, compositefile, newfile) + + insert_cmd(dir, oldfile, newfile, cmd, url, name, tag) + +else: + bin_convert(WORKING_DIR, cmd, oldfile, newfile) + insert_cmd(dir, oldfile, newfile, cmd, url, name, tag) + +# jsonp callback +if param['callback'] is not None: + url = (BASE_URL+dir+"/"+newfile).replace("'", "\\'") + width, height = bin_identify (WORKING_DIR, newfile) + print param['callback'] + "({" + print "'url':'" + url + "'," + print "'size':" + str(file_size (WORKING_DIR, newfile)) + "," + print "'width':" + width + "," + print "'height':" + height + print "});" + moveToS3(os.path.join(WORKING_DIR, newfile), "im/"+dir+"/"+newfile) +else: + print "#@im" + print ", ".join([k+"="+str(v) for k,v in param.iteritems()]) + print " ".join(cmd) + print BASE_URL+dir+"/"+newfile + print file_size (WORKING_DIR, newfile) + print bin_identify (WORKING_DIR, newfile) + moveToS3(os.path.join(WORKING_DIR, newfile), "im/"+dir+"/"+newfile) + +#remove(newfile) + +g.close() + diff --git a/pbserver.py b/pbserver.py index 5501f54..e7f190a 100755 --- a/pbserver.py +++ b/pbserver.py @@ -18,6 +18,9 @@ from subprocess import call, Popen, PIPE import simplejson as json BIN_IDENTIFY = "/usr/bin/identify" +from boto.s3.connection import S3Connection +from boto.s3.key import Key + try: DB = db.db () except Exception as e: @@ -49,21 +52,18 @@ def cleanup(filepath): sys.stderr.write(str(e)) raise -def s3move(filepath,objectname): - conn = s3.AWSAuthConnection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) - filedata = open(filepath, 'rb').read() - content_type = mimetypes.guess_type(filepath)[0] +def s3move(filename,objectname): try: - conn.put(BUCKET_NAME, objectname, s3.S3Object(filedata), - { - 'x-amz-acl': 'public-read', - 'Content-Type': content_type or 'text/plain', - 'x-amz-storage-class': 'REDUCED_REDUNDANCY' - } - ); + conn = S3Connection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, is_secure=False) + b = conn.get_bucket(BUCKET_NAME) + k = Key(b) + k.key = objectname + k.set_contents_from_filename(filename) + k.set_acl('public-read') + k.storage_class = 'REDUCED_REDUNDANCY' except Exception as e: - sys.stderr.write(str(e)) - raise + sys.stderr.write(str(e)); + raise(e) def insert_cmd (date, remote_addr, username, url, directory, oldfile, newfile, cmd, dataobj, tag): try: @@ -143,7 +143,7 @@ class Location: class AWSAuthConnection: def __init__(self, aws_access_key_id, aws_secret_access_key, is_secure=True, - server=DEFAULT_HOST, port=None, calling_format=CallingFormat.SUBDOMAIN): + server=DEFAULT_HOST, port=443, calling_format=CallingFormat.SUBDOMAIN): if not port: port = PORTS_BY_SECURITY[is_secure] |