diff options
| -rw-r--r-- | photoblaster/db/__init__.py | 17 | ||||
| -rw-r--r-- | photoblaster/db/models/__init__.py | 26 | ||||
| -rw-r--r-- | photoblaster/server.py | 21 | ||||
| -rw-r--r-- | share/frontend/imlandscape/js/pb.js | 4 | ||||
| -rw-r--r-- | sync_iasdfus_deleted.py | 38 |
5 files changed, 47 insertions, 59 deletions
diff --git a/photoblaster/db/__init__.py b/photoblaster/db/__init__.py index a3bb960..5847545 100644 --- a/photoblaster/db/__init__.py +++ b/photoblaster/db/__init__.py @@ -3,13 +3,16 @@ from sqlalchemy.orm import sessionmaker, scoped_session from photoblaster.config import DB_HOST, DB_USER, DB_PASSWORD, DB_NAME -engine = create_engine('mysql://{}:{}@{}/{}'.format( - DB_USER, - DB_PASSWORD, - DB_HOST, - DB_NAME -)) +engine = create_engine( + 'mysql://{}:{}@{}/{}'.format( + DB_USER, + DB_PASSWORD, + DB_HOST, + DB_NAME + ), + pool_recycle=3600 +) session_factory = sessionmaker(bind=engine) SessionHeap = scoped_session(session_factory) -Session = session_factory +session = session_factory() diff --git a/photoblaster/db/models/__init__.py b/photoblaster/db/models/__init__.py index a30383a..85784e0 100644 --- a/photoblaster/db/models/__init__.py +++ b/photoblaster/db/models/__init__.py @@ -1,25 +1,20 @@ -from photoblaster.db import SessionHeap, Session +from photoblaster.db import SessionHeap, session from sqlalchemy import inspect, desc from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.sql.expression import func - Base = declarative_base() - class Actions(object): @classmethod def create(cls, **kwargs): - session = Session() + global session try: session.add(cls(**kwargs)) session.commit() except: session.rollback() raise - finally: - session.close() - SessionHeap.remove() def update(self, **kwargs): for key, val in kwargs.iteritems(): @@ -31,14 +26,9 @@ class Actions(object): session.rollback() raise - def free(self): - session = inspect(self).session - session.close() - SessionHeap.remove() - @classmethod def _search(cls, **kwargs): - session = Session() + global session query = session.query(cls).filter_by(**kwargs) session.close() return query @@ -47,9 +37,6 @@ class Actions(object): except: session.rollback() raise - finally: - session.close() - SessionHeap.remove() return query @classmethod @@ -62,19 +49,14 @@ class Actions(object): @classmethod def query(cls, **kwargs): - session = SessionHeap() - print "session() query" + global session query = session.query(cls) try: session.add(cls(**kwargs)) session.commit() except: - print "session.rollback()" session.rollback() raise - finally: - session.close() - SessionHeap.remove() return query from photoblaster.db.models.imcmd import ImCmd diff --git a/photoblaster/server.py b/photoblaster/server.py index 0e89886..4a80fd0 100644 --- a/photoblaster/server.py +++ b/photoblaster/server.py @@ -81,7 +81,6 @@ class InvalidUsage(Exception): class Server(object): """Main server class""" def __init__(self): - # self.app = Flask(__name__) self.app = Flask(__name__, static_folder=STATIC_FOLDER) self._wsgi_server = None self._classname_aliases = _CLASSNAME_ALIASES @@ -90,6 +89,18 @@ class Server(object): def test(): return "HELLO WORLD!" + @self.app.route('/im/proxy', methods=['GET']) + def p_image(): + sys.stderr.write("got request") + url = request.args.get("url") + req = urllib2.Request(url=url) + req = urllib2.urlopen(req) + header = req.headers.getheader('content-type') + if re.match(r'image', header, re.IGNORECASE): + return req.read() + else: + raise InvalidUsage('Improper Usage', status_code=410) + @self.app.route('/im/api/<pb_classname>', methods=['POST']) def pb(pb_classname): x_forwarded_headers = request.headers.getlist("X-Forwarded-For") @@ -117,6 +128,7 @@ class Server(object): # send_static_file will guess the correct MIME type return self.app.send_static_file(path) + @self.app.route('/proxy', methods=['GET']) def proxy_image(): url = request.args.get("url") @@ -215,15 +227,16 @@ class Server(object): raise InvalidUsage('No such api', status_code=410) def _response_post(self, pb_classname, request_form, remote_addr=None): + #load plugins here pb_class = self._find_class_by_name(pb_classname) - # classnames = map(lambda c: c.__name__, Pb.__subclasses__()) try: pb = pb_class(**request_form) pb.create() if not LOCAL: - pb.file_s3move() + pb.get_output_file().s3move() pb.db_send(remote_addr=remote_addr) - json_data = jsonify(pb.file_dict()) + json_data = jsonify(pb.get_output_file().as_dict()) + pb.cleanup() if pb.params.callback: # accounts for jsonp return "%s(%s)" % (pb.params.callback, json_data) return json_data diff --git a/share/frontend/imlandscape/js/pb.js b/share/frontend/imlandscape/js/pb.js index 9cf8df5..55e30ac 100644 --- a/share/frontend/imlandscape/js/pb.js +++ b/share/frontend/imlandscape/js/pb.js @@ -57,8 +57,8 @@ function loadNew() { console.log(textureURL); console.log(heightmapURL); stop_animating(); - var new_texture = '/proxy?url='+textureURL; - var new_heightmap = '/proxy?url='+heightmapURL; + var new_texture = '/im/proxy?url='+textureURL; + var new_heightmap = '/im/proxy?url='+heightmapURL; initGraphics(new_texture, new_heightmap, function(){ animate() } ); } diff --git a/sync_iasdfus_deleted.py b/sync_iasdfus_deleted.py index b654c0d..c0c4bac 100644 --- a/sync_iasdfus_deleted.py +++ b/sync_iasdfus_deleted.py @@ -4,7 +4,6 @@ import re from photoblaster.db.models import Iasdfus from photoblaster.db.models import ImCmd -#database = Database() def super_unquote(s): for i in xrange(0,20): @@ -14,18 +13,10 @@ def super_unquote(s): #searches for elements in the Iasdfus table that have deleted=1 #stores all the objects as a list in memory, there are 92,000 deleted_urls = Iasdfus.search(deleted=True).all() -print len(deleted_urls) -#well do you think I should try to study the flask-sqlalchemy sources and look for clues? +#print len(deleted_urls) #well i'm reading doc on it, they recommend create one session per all requests, and we here are creating new session per #each request, not sure if that matters I guess it does. #so in other words, in the flask-sqlalchemy pattern the session is created when the server is started -#and in our case, we are wrapping a session into each query -#so instead we need to invoke session as part of this script? -#well almost. I think need to make main class like Database, and use it like this -# and inside .rs() get it froem Database() object -# I think we need to stop trying to create new sessions every time, obviously there's -# something under the hood making it impossible to use that way in any scalable situation, you know? yeah -#so instead of calling sqlalchemy.session directly in the script #instantiate pb.db.Database() #and at the end of the script #database.close(), something like this? close() can be in destroy method for dataabase, it will go out of scope at the end of script @@ -38,7 +29,7 @@ print len(deleted_urls) #im/ff/wigglelogo_1347403794_frankhats_1347403811_frankhats.gif n = 0 for url in deleted_urls: - print "from iasdfus: (%s)" % (url.address) +# print "from iasdfus: (%s)" % (url.address) #iterates through try: parts = url.address.split("/") @@ -48,16 +39,15 @@ for url in deleted_urls: newfile = super_unquote(parts[2]) newfile_parts = re.split(r'\+?http', newfile) newfile = newfile_parts[0] - print "parts from iasdfus (newfile : %s, dir: %s)" % (newfile, dirpart) +# print "parts from iasdfus (newfile : %s, dir: %s)" % (newfile, dirpart) except IndexError: continue try: #searches ImCmd for that row, where dirpart and newfile are the same as the #values in the Iasdfus row - print "imcmd: %s %s\n" % (dirpart, newfile) #k so looks like it's stuck here regardless on item, so this probably session limit, or something like that - query = database.rs('ImCmd').search(**{"dir": dirpart, "newfile": newfile}) - print "imcmd done query n: %d\n" % (n) + query = ImCmd.search(**{"dir": dirpart, "newfile": newfile}) +# print "imcmd done query n: %d\n" % (n) n +=1 #can't find anything about it me too #so it doesn't have .free(), I think it doesni so it's some sort of session parameter? looks so seen it anywhere? have free only when it doesn't find anything it's still a Query objects @@ -66,16 +56,16 @@ for url in deleted_urls: #it seems like my theory might be right...seems like it to you right? not sure yet #processes the query matching_url = query.first() - print "got first\n" - #if not matching_url: - # print "nothing found\n" - # continue - #if matching_url.deleted == 1: - # print "is deleted\n" - # continue +# print "got first\n" + if not matching_url: + print "imcmd: %s %s\n" % (dirpart, newfile) + continue + if matching_url.deleted == 1: +# print "is deleted\n" + continue #update matching_url - #matching_url.update(deleted=1) - #print "update done\n" + matching_url.update(deleted=1) + print "update done\n" except AttributeError: raise #honestly can't imagine why I'm getting this error maybe we just need to implement another rollback exception? |