From 8557402d4cca9d2a99ba344310c494ee17a988c6 Mon Sep 17 00:00:00 2001 From: Pepper Date: Thu, 30 Apr 2015 20:01:33 -0400 Subject: looking good --- generate | 376 ------------------------------------------------------------ generate.py | 290 ++++++++++++++++++++++++++++++++++++++++++++++ pbserver.py | 16 +-- 3 files changed, 294 insertions(+), 388 deletions(-) delete mode 100755 generate create mode 100755 generate.py diff --git a/generate b/generate deleted file mode 100755 index 0585b10..0000000 --- a/generate +++ /dev/null @@ -1,376 +0,0 @@ -#!/usr/bin/python2.7 -import sys -import os -import re -import time -import string -import urllib, urllib2 -from subprocess import Popen, PIPE -import sha -import simplejson as json - -import mimetypes - -urlencode = urllib.urlencode -urlopen = urllib2.urlopen -Request = urllib2.Request - -MAX_SIZE = 1024 * 1024 * 1.2 * 1.5 -WORKING_DIR = "/var/www/cache" -LIKE_A_BOSS = "ryz pepper seamonkey JAMES".split(" ") - -BIN_CONVERT = "/usr/bin/convert" -BIN_COMPOSITE = "/usr/bin/composite" -BIN_IDENTIFY = "/usr/bin/identify" -DEFAULT_FINALFORMAT = "gif" -DB_TAG = "pb"; - -GRAVITY = "NorthWest North NorthEast West Center East SouthWest South SouthEast".split(" ") - -#{{{Utility functions -def is_color(s): - if s == "": - return "transparent" - if re.match('(rgba?\([0-9]+,[0-9]+,[0-9]+\))|([a-zA-Z]+)|(\#[A-Ha-h0-9]+)', s): - return s.replace(' ', ''); - else: - sys.stderr.write("Not a color: {}\n".format(s)) - raise ValueError -def bool_correct(s): - if re.match(r'^false$', s, re.IGNORECASE): - return False - elif re.match(r'^true$', s, re.IGNORECASE): - return True - else: - return s - -class dotdict(dict): - """dot.notation access to dictionary attributes""" - def __getattr__(self, attr): - return self.get(attr) - __setattr__= dict.__setitem__ - __delattr__= dict.__delitem__ - -def get_mimetype(f): - try: - mimetype = Popen( - [BIN_IDENTIFY, f], stdout=PIPE - ).communicate()[0].split(" ")[1].lower() - return mimetype - except Exception as e: - sys.stderr.write("IMGRID couldn't determine mimetype") - sys.stderr.write(str(e)) - raise; - -def sanitize (str): - return re.sub(r'\W+', '', str) - -def now(): - return int(time.time()) - -def browser_request (url, data=None): - headers = { - 'User-Agent': 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)', - 'Accept': '*/*', - } - try: - req = Request(url, data, headers) - response = urlopen(req) - except IOError, e: - if hasattr(e, 'code'): - sys.stderr.write( '%s - ERROR %s' % (url, e.code) ) - raise; - return None - else: - return response - -def download(url, destination, max_size=MAX_SIZE): - response = browser_request(url, None) - rawimg = response.read() - if len(rawimg) == 0: - sys.stderr.write("got zero-length file") - raise; - if len(rawimg) > max_size: - sys.stderr.write("file too big: max size {} KB / {} is {} KB".format( - str(MAX_SIZE/1024), - destination, - str(len(rawimg)/1024) - )) - raise; - f = open(destination, "w") - f.write(rawimg) - f.close() - -def file_size (filepath): - try: - return os.stat(file)[6] - except Exception as e: - sys.stderr.write("IMGRID couldn't determine file size") - sys.stderr.write(str(e)) - raise; - -def gif_frames(filepath): - try: - info = Popen([BIN_IDENTIFY,filepath], stdout=PIPE).communicate()[0] - frames = filter((lambda x: x), map( - (lambda x: x.split(" ")[0]), - (info).split('\n') - )) - return frames - except Exception as e: - sys.stderr.write("IMGRID couldn't get gif frames") - sys.stderr.write(str(e)) - raise; -#}}} - -class Photoblaster(): - def __init__(self, params): - self.params = {} - self.tag = "im" - self.now = now() - self.files_created = [] - self.commands = []; - self._required_keys = [ -#{{{ required_keys - "url", - "transparent", - "subtract", - "fuzz", - "width", - "height", - "black", - "white", - "brightness", - "contrast", - "saturation", - "hue", - "rotate", - "flip", - "flop", - "background", - "merge_early", - "compose", - "gravity", - "tile", - "format", - "name", - "callback", - "coalesce", - "nearest", - "dispose", -#}}} - ] - for k in self._required_keys: - if k in params: - if k in [ 'url', 'background' ] and bool_correct(params[k]): - self.params[k] = {} - self.params[k]['url'] = params[k] - self.params[k]['filename'] = "PBTMP{}_{}".format(now(), k) - - self.params[k]['path'] = os.path.join(WORKING_DIR, self.params[k]['filename']) - try: - download(self.params[k]['url'], self.params[k]['path']) - self.files_created.append(self.params[k]['path']) - self.params[k]['mimetype'] = get_mimetype(self.params[k]['path']) - frames = gif_frames(self.params[k]['path']) - if len(frames) > 1: - self.params[k]['path'] = random.choice(frames) - - - except Exception as e: - sys.stderr.write("BAD PARAMS\n") - sys.stderr.write(str(e)) - raise; - elif k in [ 'black', 'white', 'subtract' ]: - try: - self.params[k] = is_color(params[k]) - except Exception as e: - sys.stderr.write("Unable to process color for:\n") - sys.stderr.write(k) - raise e - else: - self.params[k] = bool_correct(sanitize(params[k])) - else: - self.params[k] = False; - - self.params = dotdict(self.params) - - self.basename = self._get_filename(); - - if not self.params.format: - self.params.format = DEFAULT_FINALFORMAT - self.filename = "{}.{}".format(self.basename, self.params.format) - #final filepath is stored in self.filepath - self.filepath = os.path.join(WORKING_DIR, self.filename) - - def _get_filename(self): - return "{}_{}_{}".format( - self.tag, - now(), - self.params.username or "" - ); - - def _call_cmd(self, cmd, error=""): - try: - call(cmd) - self.commands.append(" ".join(cmd)); - except Exception as e: - sys.stderr.write("IT HIT AN ERROR") - sys.stderr.write(str(cmd)) - if error: - sys.stderr.write(error) - else: - sys.stderr.write(str(e)) - - def _cleanup(self): - if not len(self.files_created): - pass - cmd = ["rm", "-f"] + self.files_created - self._call_cmd(cmd) - -def bin_composite (params, bottomdir, bottomfile, topdir, topfile, newfile): - cmd = [BIN_CONVERT] - cmd.append(os.path.join(bottomdir,bottomfile)) - cmd.append('null:') - cmd.append(os.path.join(topdir,topfile)) - cmd.append("-matte") - cmd.extend(params) - cmd.append("-layers composite") - cmd.append(os.path.join(topdir, newfile)) -# g.write("bin_composite command: %s" % (" ".join(cmd))); - os.system(" ".join(cmd)) - os.system("rm "+os.path.join(bottomdir,bottomfile)+" "+os.path.join(topdir,topfile)) - -def bin_convert (directory, params, oldfile, newfile): - cmd = [BIN_CONVERT, os.path.join(directory, oldfile)] - for x in params: - cmd.append(x) - cmd.append(os.path.join(directory,newfile)) - os.system(" ".join(cmd)) -# g.write("bin_convert command: %s" % (" ".join(cmd))); - os.system("rm "+os.path.join(directory,oldfile)) - - def _build_cmd(self): - cmd = [] - #FIXME test if number - if self.params.rotate: cmd += ["-rotate", self.params.rotate ] - if self.params.flip: cmd += ["-flip"] - if self.params.flop: cmd += ["-flop"] - -if param['transparent'] == "true": - tag = "transparent" - if is_number(param['fuzz']) and param['fuzz'] != 0: - cmd.append("-fuzz") - cmd.append(param['fuzz']+"%") - subtract_color = as_color(param['subtract'], "white") - cmd.append("-transparent") - cmd.append(subtract_color) - -VALID_DISPOSE_METHODS=["none","previous","background"] -dispose = "None" - -if param['width'] is not None and is_number(param['width']): - if int(param['width']) > 1000 and NOT_A_BOSS: - error ("width cannot be greater than 1000") - width = param['width'] -if param['height'] is not None and is_number(param['height']): - if int(param['height']) > 1000 and NOT_A_BOSS: - error ("height cannot be greater than 1000") - height = param['height'] - -if (width or height): - if param['nearest'] == "true": - if format == "gif": - cmd.append("-coalesce") - cmd.append("+map") - cmd.append("-interpolate") - cmd.append("Nearest") - cmd.append("-interpolative-resize") - else: - cmd.append("-resize") - -if width and height: - cmd.append(width + "x" + height) -elif width: - cmd.append(width) -elif height: - cmd.append("x" + height) - -if param['black'] != "black" or param['white'] != 'white': - try: - black = as_color(param['black'], "black") - white = as_color(param['white'], "white") - cmd.append("+level-colors") - cmd.append(black+","+white) - except (): - pass - -if param['contrast'] is not None and is_number(param['contrast']): - cmd.append("-contrast-stretch") - cmd.append(param['contrast']) - pass - -if param['brightness'] is not None or param['saturation'] is not None or param['hue'] is not None: - bstring = '' - if is_number(param['brightness']): - bstring += param['brightness'] - else: - bstring += "100" - bstring += ',' - if is_number(param['contrast']): - bstring += param['contrast'] - else: - bstring += "100" - bstring += ',' - if is_number(param['hue']): - bstring += param['hue'] - if bstring != "100,100,": - cmd.append("-modulate") - cmd.append(bstring) - -if bgfile is not None: - tag = param['compose'] - gravity = param['gravity'] - - if gravity not in GRAVITY: - gravity = 'center' - - - compositefile = "composite_" + newfile - compositeparams = ["-dispose", "None", "-gravity", gravity] - compositeparams.extend([ "-compose", param['compose'] ]) - - cmd.append( "-coalesce") - bin_convert (WORKING_DIR, cmd, oldfile, compositefile) - bin_composite (compositeparams, WORKING_DIR, bgfile, WORKING_DIR, compositefile, newfile) - - insert_cmd(dir, oldfile, newfile, cmd, url, name, tag) - -else: - bin_convert(WORKING_DIR, cmd, oldfile, newfile) - insert_cmd(dir, oldfile, newfile, cmd, url, name, tag) - -# jsonp callback -if param['callback'] is not None: - url = (BASE_URL+dir+"/"+newfile).replace("'", "\\'") - width, height = bin_identify (WORKING_DIR, newfile) - print param['callback'] + "({" - print "'url':'" + url + "'," - print "'size':" + str(file_size (WORKING_DIR, newfile)) + "," - print "'width':" + width + "," - print "'height':" + height - print "});" - moveToS3(os.path.join(WORKING_DIR, newfile), "im/"+dir+"/"+newfile) -else: - print "#@im" - print ", ".join([k+"="+str(v) for k,v in param.iteritems()]) - print " ".join(cmd) - print BASE_URL+dir+"/"+newfile - print file_size (WORKING_DIR, newfile) - print bin_identify (WORKING_DIR, newfile) - moveToS3(os.path.join(WORKING_DIR, newfile), "im/"+dir+"/"+newfile) - -#remove(newfile) - -g.close() - diff --git a/generate.py b/generate.py new file mode 100755 index 0000000..923b39a --- /dev/null +++ b/generate.py @@ -0,0 +1,290 @@ +#!/usr/bin/python2.7 +import sys +import os +import re +import sha + +from pb.config import * +import pb.lib.utils as utils + +LIKE_A_BOSS = "ryz pepper seamonkey JAMES".split(" ") +DEFAULT_FINALFORMAT = "gif" +DB_TAG = "pb"; + +GRAVITY = "NorthWest North NorthEast West Center East SouthWest South SouthEast".split(" ") +FORMAT = "jpg gif png".split(" ") +COMPOSE = "Over ATop Dst_Over Dst_In Dst_Out Multiply Screen Divide Plus Difference Exclusion "+ + "Lighten Darken Overlay Hard_Light Soft_Light Pegtop_Light Linear_Light Vivid_Light Pin_Light "+ + "Linear_Dodge Linear_Burn Color_Dodge Color_Burn".split(" "); + +class Generate(): + def __init__(self, **kwargs): + self.params = {} + self.tag = "im" + self.now = utils.now() + self.files_created = [] + self.commands = []; + self._required_keys = [ +#{{{ required_keys + #IMAGES + "url", + "background", + + #BOOLS + "coalesce", + "dispose", + "nearest", + "merge_early", + "flip", + "flop", + "tile", + "transparent", + + #COLORS + "black", + "white", + "subtract", + + #INTS + "fuzz", + "width", + "height", + "brightness", + "contrast", + "saturation", + "rotate", + "hue", + + #ENUMS + "compose", + "gravity", + "format", + + #STRINGS + "name", + "callback", +#}}} + ] + for k in self._required_keys: + if k in kwargs: + if k in [ 'url', 'background' ] and utils.bool_correct(kwargs[k]): + self.params[k] = { + 'url' : kwargs[k], + 'filename' : self._make_tempname(k), + 'path' : os.path.join(WORKING_DIR, self._make_tempname(k)) , + } + try: + utils.download(self.params[k]['url'], self.params[k]['path']) + self.files_created.append(self.params[k]['path']) + self.params[k]['mimetype'] = utils.get_mimetype(self.params[k]['path']) + except Exception: + sys.stderr.write(str(e)) + raise Exception ("BAD PARAMS"); + elif k in [ 'black', 'white', 'subtract' ]: + try: + self.params[k] = utils.is_color(kwargs[k]) + except Exception: + raise Exception("Unable to process color for:\n{}".format(k)) + elif k in [ + "coalesce", "dispose", "nearest", "merge_early", + "flip", "flop", "tile", "transparent", + ]: + self.params[k] = utils.bootl_correct(utils.sanitize(kwargs[k])) + elif k == 'gravity' and self._test_enum(k, GRAVITY): + self.params[k] = kwargs[k] + elif k == 'format' and self._test_enum(k, FORMAT): + self.params[k] = kwargs[k] + elif k == 'compose' and self._test_enum(k, COMPOSE): + self.params[k] = kwargs[k] + elif k in [ "fuzz", "width", "height", "brightness", "contrast", "saturation", "rotate", "hue" ]: + self.params[k] = int(params[k]) + else: + self.params[k] = utils.sanitize(kwargs[k]) + else: + self.params[k] = None; + + self.params = utils.dotdict(self.params) + + self.basename = self._get_filename(); + + if not self.params.format: + self.params.format = DEFAULT_FINALFORMAT + self.filename = "{}.{}".format(self.basename, self.params.format) + #final filepath is stored in self.filepath + self.filepath = os.path.join(WORKING_DIR, self.filename) + + def _make_tempname(self, s): + return "PBTMP{}{}".format(self.now, s); + + def _test_enum(self, k, arr): + if k in arr: + return True + else: + raise Exception("Bad Param: {}".format(k)) + + def _get_filename(self): + return "{}_{}_{}".format( + self.tag, + now(), + self.params.username or "" + ); + + def _call_cmd(self, cmd): + try: + utils.call_cmd(cmd) + self.commands.append(" ".join(cmd)); + except Exception: + raise Exception("Unable to call cmd {}".format(str(cmd))) + + def _cleanup(self): + if not len(self.files_created): + pass + cmd = ["rm", "-f"] + self.files_created + self._call_cmd(cmd) + +def bin_composite (params, bottomdir, bottomfile, topdir, topfile, newfile): + cmd = [BIN_CONVERT] + cmd.append(os.path.join(bottomdir,bottomfile)) + cmd.append('null:') + cmd.append(os.path.join(topdir,topfile)) + cmd.append("-matte") + cmd.extend(params) + cmd.append("-layers composite") + cmd.append(os.path.join(topdir, newfile)) +# g.write("bin_composite command: %s" % (" ".join(cmd))); + os.system(" ".join(cmd)) + os.system("rm "+os.path.join(bottomdir,bottomfile)+" "+os.path.join(topdir,topfile)) + +def bin_convert (directory, params, oldfile, newfile): + cmd = [BIN_CONVERT, os.path.join(directory, oldfile)] + for x in params: + cmd.append(x) + cmd.append(os.path.join(directory,newfile)) + os.system(" ".join(cmd)) +# g.write("bin_convert command: %s" % (" ".join(cmd))); + os.system("rm "+os.path.join(directory,oldfile)) + + def _build_cmd(self): + cmd = [] + #FIXME test if number + if self.params.rotate: cmd += ["-rotate", self.params.rotate ] + if self.params.flip: cmd += ["-flip"] + if self.params.flop: cmd += ["-flop"] + +if param['transparent'] == "true": + tag = "transparent" + if is_number(param['fuzz']) and param['fuzz'] != 0: + cmd.append("-fuzz") + cmd.append(param['fuzz']+"%") + subtract_color = as_color(param['subtract'], "white") + cmd.append("-transparent") + cmd.append(subtract_color) + +VALID_DISPOSE_METHODS=["none","previous","background"] +dispose = "None" + +if param['width'] is not None and is_number(param['width']): + if int(param['width']) > 1000 and NOT_A_BOSS: + error ("width cannot be greater than 1000") + width = param['width'] +if param['height'] is not None and is_number(param['height']): + if int(param['height']) > 1000 and NOT_A_BOSS: + error ("height cannot be greater than 1000") + height = param['height'] + +if (width or height): + if param['nearest'] == "true": + if format == "gif": + cmd.append("-coalesce") + cmd.append("+map") + cmd.append("-interpolate") + cmd.append("Nearest") + cmd.append("-interpolative-resize") + else: + cmd.append("-resize") + +if width and height: + cmd.append(width + "x" + height) +elif width: + cmd.append(width) +elif height: + cmd.append("x" + height) + +if param['black'] != "black" or param['white'] != 'white': + try: + black = as_color(param['black'], "black") + white = as_color(param['white'], "white") + cmd.append("+level-colors") + cmd.append(black+","+white) + except (): + pass + +if param['contrast'] is not None and is_number(param['contrast']): + cmd.append("-contrast-stretch") + cmd.append(param['contrast']) + pass + +if param['brightness'] is not None or param['saturation'] is not None or param['hue'] is not None: + bstring = '' + if is_number(param['brightness']): + bstring += param['brightness'] + else: + bstring += "100" + bstring += ',' + if is_number(param['contrast']): + bstring += param['contrast'] + else: + bstring += "100" + bstring += ',' + if is_number(param['hue']): + bstring += param['hue'] + if bstring != "100,100,": + cmd.append("-modulate") + cmd.append(bstring) + +if bgfile is not None: + tag = param['compose'] + gravity = param['gravity'] + + if gravity not in GRAVITY: + gravity = 'center' + + + compositefile = "composite_" + newfile + compositeparams = ["-dispose", "None", "-gravity", gravity] + compositeparams.extend([ "-compose", param['compose'] ]) + + cmd.append( "-coalesce") + bin_convert (WORKING_DIR, cmd, oldfile, compositefile) + bin_composite (compositeparams, WORKING_DIR, bgfile, WORKING_DIR, compositefile, newfile) + + insert_cmd(dir, oldfile, newfile, cmd, url, name, tag) + +else: + bin_convert(WORKING_DIR, cmd, oldfile, newfile) + insert_cmd(dir, oldfile, newfile, cmd, url, name, tag) + +# jsonp callback +if param['callback'] is not None: + url = (BASE_URL+dir+"/"+newfile).replace("'", "\\'") + width, height = bin_identify (WORKING_DIR, newfile) + print param['callback'] + "({" + print "'url':'" + url + "'," + print "'size':" + str(file_size (WORKING_DIR, newfile)) + "," + print "'width':" + width + "," + print "'height':" + height + print "});" + moveToS3(os.path.join(WORKING_DIR, newfile), "im/"+dir+"/"+newfile) +else: + print "#@im" + print ", ".join([k+"="+str(v) for k,v in param.iteritems()]) + print " ".join(cmd) + print BASE_URL+dir+"/"+newfile + print file_size (WORKING_DIR, newfile) + print bin_identify (WORKING_DIR, newfile) + moveToS3(os.path.join(WORKING_DIR, newfile), "im/"+dir+"/"+newfile) + +#remove(newfile) + +g.close() + diff --git a/pbserver.py b/pbserver.py index fb7fb65..a2fb1a7 100755 --- a/pbserver.py +++ b/pbserver.py @@ -6,16 +6,16 @@ from pb.imgrid import Imgrid from pb.breaker import Breaker from pb.pattern import Pattern -from pb.config import AWS_SECRET_ACCESS_KEY, AWS_ACCESS_KEY_ID, BUCKET_NAME +from pb.config import AWS_SECRET_ACCESS_KEY, AWS_ACCESS_KEY_ID, BUCKET_NAME, BIN_IDENTIFY +import pb.lib.utils as utils import os import sys import pb.lib.db as db -import mimetypes + import sha from subprocess import call, Popen, PIPE import simplejson as json -BIN_IDENTIFY = "/usr/bin/identify" from boto.s3.connection import S3Connection from boto.s3.key import Key @@ -30,20 +30,12 @@ BASE_URL = "http://i.asdf.us" def hashdir(filename): return sha.new(filename).hexdigest()[:2] -def file_size (filepath): - try: - return os.stat(filepath)[6] - except Exception as e: - sys.stderr.write(str(e)) - raise; - def bin_identify (filepath): ident = Popen([BIN_IDENTIFY, filepath], stdout=PIPE).communicate()[0] partz = ident.split(" ") width,height = partz[2].split("x") return [ width, height ] - def cleanup(filepath): try: call(['rm', filepath]) @@ -80,7 +72,7 @@ def insert_cmd (date, remote_addr, username, url, directory, oldfile, newfile, c def return_image(im, insert_url="NULL"): directory = hashdir(im.filename) dimensions = bin_identify(im.filepath) - size = file_size(im.filepath) + size = utils.file_size(im.filepath) objectname = "im/{}/{}".format(directory, im.filename) try: s3move(im.filepath, objectname) -- cgit v1.2.3-70-g09d2