diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/fill.image_altars.py | 2 | ||||
| -rwxr-xr-x | scripts/fill.image_urls.py | 2 | ||||
| -rwxr-xr-x | scripts/idgi/filenames.list.txt | 191 | ||||
| -rwxr-xr-x | scripts/logreport.py | 32 |
4 files changed, 112 insertions, 115 deletions
diff --git a/scripts/fill.image_altars.py b/scripts/fill.image_altars.py index 1181214..3d248a1 100755 --- a/scripts/fill.image_altars.py +++ b/scripts/fill.image_altars.py @@ -27,7 +27,7 @@ def add_altar(message_id, user_id, content): except postgresql.exceptions.UniqueError: print("skipped adding a dupe") -# NOTE. hardcoded room numbers to index here... only indexing DUMPFM (1) and GIF (8) currently. +# NOTE: hardcoded room numbers to index here... only indexing HUMPFM (1) and GIF (8) currently. def get_messages(): ps = db.prepare("SELECT message_id, user_id, content FROM messages WHERE message_id >= $1 AND message_id <= $2 AND room_id IN (1,8) ORDER BY message_id ASC") return ps.chunks diff --git a/scripts/fill.image_urls.py b/scripts/fill.image_urls.py index d3a10de..a7c518b 100755 --- a/scripts/fill.image_urls.py +++ b/scripts/fill.image_urls.py @@ -29,7 +29,7 @@ def add_url(url): ps2 = db.prepare("INSERT INTO image_urls(url) VALUES($1)") ps2(url) -# NOTE. hardcoded room numbers to index here... only indexing DUMPFM (1) and GIF (8) currently. +# NOTE: hardcoded room numbers to index here... only indexing HUMPFM (1) and GIF (8) currently. def get_messages(lower, upper): ps = db.prepare("SELECT content FROM messages WHERE message_id >= $1 AND message_id <= $2 AND room_id IN (1,8) ORDER BY message_id ASC") rows = ps(lower, upper) diff --git a/scripts/idgi/filenames.list.txt b/scripts/idgi/filenames.list.txt index a3776f1..cc0c19e 100755 --- a/scripts/idgi/filenames.list.txt +++ b/scripts/idgi/filenames.list.txt @@ -1,98 +1,97 @@ images/1266097108677-notgoingtohappen.jpg images/1266255579876-dontgetit.png -images/20100312/1268412932681-dumpfm-andrej-this-guy-knows.gif -images/20100418/1271584646128-dumpfm-whatevz-ist2_276319-corporate-your-message-here.jpg -images/20100329/1269836545708-dumpfm-ryder-usdancing.gif -images/20100329/1269836638081-dumpfm-ryder-dudedancing.gif -images/20100330/1269924366027-dumpfm-frankhats-bedunno.jpg -images/20100330/1269930997328-dumpfm-ryder-Picture-41.png -images/20100331/1270082757023-dumpfm-jeanette-Picture-3.png -images/20100401/1270100522426-dumpfm-ryder-Picture-90.png -images/20100401/1270179079682-dumpfm-ryder-Picture-114.png -images/20100401/1270179478809-dumpfm-ryder-wolfguy.gif -images/20100402/1270195156779-dumpfm-frankhats-dunliq.jpg -images/20100402/1270196318946-dumpfm-illalli-hothat.jpg -images/20100401/1270151653306-dumpfm-ryder-Picture-108.png -images/20100404/1270371704503-dumpfm-frankhats-crudun.gif -images/20100404/1270436206347-dumpfm-frankhats-dungetilitter.gif -images/20100405/1270449719939-dumpfm-illalli-likemike.jpg -images/20100405/1270451213193-dumpfm-ryder-Picture-187.png -images/20100405/1270453310699-dumpfm-ryder-yruignoring.gif -images/20100405/1270454430952-dumpfm-frankhats-dundora.jpg -images/20100407/1270677806850-dumpfm-frankhats-gildedgetit.jpg -images/20100415/1271321993419-dumpfm-ryder-Picture-452.png -images/20100407/1270697865705-dumpfm-ryder-artpwnd.gif -images/20100407/1270698050400-dumpfm-ryder-dumpfmfresh.gif -images/20100413/1271195512043-dumpfm-noisia-dontgetarm.jpg -images/20100408/1270705138983-dumpfm-ryder-Picture-247.png -images/20100408/1270758039197-dumpfm-ryder-Picture-254.png -images/20100408/1270764920229-dumpfm-ryder-Picture-260.png -images/20100409/1270801759113-dumpfm-jertronic-nogetit.png -images/20100328/1269805908876-dumpfm-ryder-dumplift.gif -images/20100319/1268991384841-dumpfm-jeanette-Picture-540.png -images/20100410/1270872008733-dumpfm-frankhats-Picture-14.png -images/20100410/1270872200318-dumpfm-frankhats-dontcatit.jpg -images/20100410/1270873042199-dumpfm-frankhats-catrip.gif -images/20100410/1270953524877-dumpfm-timb-idgi.cat.gif -images/20100410/1270953524877-dumpfm-timb-idgi.cat.gif -images/20100411/1270962214585-dumpfm-ryder-joindumpfm.png -images/20100408/1270708774547-dumpfm-jertronic-idontevengetyou.gif +images/20100312/1268412932681-humpfm-andrej-this-guy-knows.gif +images/20100418/1271584646128-humpfm-whatevz-ist2_276319-corporate-your-message-here.jpg +images/20100329/1269836545708-humpfm-ryder-usdancing.gif +images/20100329/1269836638081-humpfm-ryder-dudedancing.gif +images/20100330/1269924366027-humpfm-frankhats-bedunno.jpg +images/20100330/1269930997328-humpfm-ryder-Picture-41.png +images/20100331/1270082757023-humpfm-jeanette-Picture-3.png +images/20100401/1270100522426-humpfm-ryder-Picture-90.png +images/20100401/1270179079682-humpfm-ryder-Picture-114.png +images/20100401/1270179478809-humpfm-ryder-wolfguy.gif +images/20100402/1270195156779-humpfm-frankhats-dunliq.jpg +images/20100402/1270196318946-humpfm-illalli-hothat.jpg +images/20100401/1270151653306-humpfm-ryder-Picture-108.png +images/20100404/1270371704503-humpfm-frankhats-crudun.gif +images/20100404/1270436206347-humpfm-frankhats-dungetilitter.gif +images/20100405/1270449719939-humpfm-illalli-likemike.jpg +images/20100405/1270451213193-humpfm-ryder-Picture-187.png +images/20100405/1270453310699-humpfm-ryder-yruignoring.gif +images/20100405/1270454430952-humpfm-frankhats-dundora.jpg +images/20100407/1270677806850-humpfm-frankhats-gildedgetit.jpg +images/20100415/1271321993419-humpfm-ryder-Picture-452.png +images/20100407/1270697865705-humpfm-ryder-artpwnd.gif +images/20100407/1270698050400-humpfm-ryder-humpfmfresh.gif +images/20100413/1271195512043-humpfm-noisia-dontgetarm.jpg +images/20100408/1270705138983-humpfm-ryder-Picture-247.png +images/20100408/1270758039197-humpfm-ryder-Picture-254.png +images/20100408/1270764920229-humpfm-ryder-Picture-260.png +images/20100409/1270801759113-humpfm-jertronic-nogetit.png +images/20100319/1268991384841-humpfm-jeanette-Picture-540.png +images/20100410/1270872008733-humpfm-frankhats-Picture-14.png +images/20100410/1270872200318-humpfm-frankhats-dontcatit.jpg +images/20100410/1270873042199-humpfm-frankhats-catrip.gif +images/20100410/1270953524877-humpfm-timb-idgi.cat.gif +images/20100410/1270953524877-humpfm-timb-idgi.cat.gif +images/20100411/1270962214585-humpfm-ryder-joinhumpfm.png +images/20100408/1270708774547-humpfm-jertronic-idontevengetyou.gif images/1267264103873-dontgetit.gif images/1266377745509-popeandguy.gif images/1266214389930-stilldontgetit.png -images/20100412/1271118251387-dumpfm-ryder-Picture-381.png -images/20100412/1271118841756-dumpfm-jertronic-idgisplode.gif -images/20100412/1271119643882-dumpfm-ryder-Picture-384.png -images/20100413/1271133664413-dumpfm-ryder-Picture-397.png +images/20100412/1271118251387-humpfm-ryder-Picture-381.png +images/20100412/1271118841756-humpfm-jertronic-idgisplode.gif +images/20100412/1271119643882-humpfm-ryder-Picture-384.png +images/20100413/1271133664413-humpfm-ryder-Picture-397.png images/1267044843246-lady.gif images/1267043111981-Picture-2.png -images/20100413/1271193181035-dumpfm-jeanette-Picture-1047.png -images/20100413/1271193218714-dumpfm-ryder-sniffycatguymason.gif -images/20100413/1271195890446-dumpfm-mat3i-HIM.gif -images/20100415/1271321054370-dumpfm-foot-Picture-53.png -images/20100413/1271195996111-dumpfm-frankhats-dontgethead.jpg -images/20100415/1271389483032-dumpfm-shaq-retard.jpg -images/20100326/1269583520221-dumpfm-ryder-Picture-2081.png -images/20100416/1271399363348-dumpfm-frankhats-dontfriends.jpg -images/20100417/1271489503232-dumpfm-frankhats-dontgetturdeating.gif -images/20100401/1270177487240-dumpfm-ryder-guyfunny.png -images/20100417/1271497185798-dumpfm-foot-ykW0S.jpg -images/20100417/1271497299338-dumpfm-frankhats-getbag.jpg -images/20100405/1270451213193-dumpfm-ryder-Picture-187.png -images/20100405/1270448791853-dumpfm-ryder-Picture-169.png -images/20100403/1270290052415-dumpfm-ryder-Picture-148.png -images/20100402/1270261868168-dumpfm-foot-idk.jpg -images/20100405/1270451552583-dumpfm-ryder-guybreadhover.gif -images/20100417/1271562875087-dumpfm-jertronic-cry-until-get-it.gif -images/20100418/1271579861201-dumpfm-foot-1271579304078-dumpfm-ryder-Picture-516-1.png -images/20100418/1271579977086-dumpfm-ryder-Picture-521.png -images/20100418/1271580228382-dumpfm-ryder-Picture-523.png -images/20100418/1271581669282-dumpfm-ryder-Picture-525.png -images/20100418/1271584909042-dumpfm-frankhats-hiding.jpg -images/20100418/1271579100256-dumpfm-pheezer-idontgetitfuckyou.gif -images/20100418/1271646064613-dumpfm-frankhats-getjobs.gif -images/20100418/1271582779905-dumpfm-frankhats-dontlook.gif -images/20100418/1271637365874-dumpfm-ryder-Picture-546.png -images/20100418/1271637177773-dumpfm-ryder-Picture-545.png -images/20100418/1271628385535-dumpfm-ryder-Picture-543.png -images/20100419/1271654870347-dumpfm-foot-Picture-44.png -images/20100418/1271618348378-dumpfm-johntransue-1271581782422-dumpfm-foot-Picture-81.gif -images/20100419/1271663567628-dumpfm-frankhats-bikeleader.jpg -images/20100418/1271584609909-dumpfm-ryder-Picture-532.png -images/20100419/1271664504149-dumpfm-foot-24044_1382683481693_1070133119_1141455_4968953_n.jpg -images/20100419/1271664900678-dumpfm-foot-1271664772627-dumpfm-jeanette-Picture-524.png -images/20100419/1271665856679-dumpfm-jeanette-Picture-1203.png -images/20100324/1269407028795-dumpfm-frankhats-duntv.jpg -images/20100323/1269334306336-dumpfm-ryder-Picture-2031.png -images/20100322/1269243170176-dumpfm-ryder-Picture-1992.png +images/20100413/1271193181035-humpfm-jeanette-Picture-1047.png +images/20100413/1271193218714-humpfm-ryder-sniffycatguymason.gif +images/20100413/1271195890446-humpfm-mat3i-HIM.gif +images/20100415/1271321054370-humpfm-foot-Picture-53.png +images/20100413/1271195996111-humpfm-frankhats-dontgethead.jpg +images/20100415/1271389483032-humpfm-shaq-retard.jpg +images/20100326/1269583520221-humpfm-ryder-Picture-2081.png +images/20100416/1271399363348-humpfm-frankhats-dontfriends.jpg +images/20100417/1271489503232-humpfm-frankhats-dontgetturdeating.gif +images/20100401/1270177487240-humpfm-ryder-guyfunny.png +images/20100417/1271497185798-humpfm-foot-ykW0S.jpg +images/20100417/1271497299338-humpfm-frankhats-getbag.jpg +images/20100405/1270451213193-humpfm-ryder-Picture-187.png +images/20100405/1270448791853-humpfm-ryder-Picture-169.png +images/20100403/1270290052415-humpfm-ryder-Picture-148.png +images/20100402/1270261868168-humpfm-foot-idk.jpg +images/20100405/1270451552583-humpfm-ryder-guybreadhover.gif +images/20100417/1271562875087-humpfm-jertronic-cry-until-get-it.gif +images/20100418/1271579861201-humpfm-foot-1271579304078-humpfm-ryder-Picture-516-1.png +images/20100418/1271579977086-humpfm-ryder-Picture-521.png +images/20100418/1271580228382-humpfm-ryder-Picture-523.png +images/20100418/1271581669282-humpfm-ryder-Picture-525.png +images/20100418/1271584909042-humpfm-frankhats-hiding.jpg +images/20100418/1271579100256-humpfm-pheezer-idontgetitfuckyou.gif +images/20100418/1271646064613-humpfm-frankhats-getjobs.gif +images/20100418/1271582779905-humpfm-frankhats-dontlook.gif +images/20100418/1271637365874-humpfm-ryder-Picture-546.png +images/20100418/1271637177773-humpfm-ryder-Picture-545.png +images/20100418/1271628385535-humpfm-ryder-Picture-543.png +images/20100419/1271654870347-humpfm-foot-Picture-44.png +images/20100418/1271618348378-humpfm-johntransue-1271581782422-humpfm-foot-Picture-81.gif +images/20100419/1271663567628-humpfm-frankhats-bikeleader.jpg +images/20100418/1271584609909-humpfm-ryder-Picture-532.png +images/20100419/1271664504149-humpfm-foot-24044_1382683481693_1070133119_1141455_4968953_n.jpg +images/20100419/1271664900678-humpfm-foot-1271664772627-humpfm-jeanette-Picture-524.png +images/20100419/1271665856679-humpfm-jeanette-Picture-1203.png +images/20100324/1269407028795-humpfm-frankhats-duntv.jpg +images/20100323/1269334306336-humpfm-ryder-Picture-2031.png +images/20100322/1269243170176-humpfm-ryder-Picture-1992.png images/1266884561915-Screen-shot-2010-02-22-at-7.22.13-PM.png images/1266866394012-Picture-1245.png -images/20100321/1269211476225-dumpfm-jeanette-Picture-590.png +images/20100321/1269211476225-humpfm-jeanette-Picture-590.png images/1266904219513-Picture-1288.png images/1266205200000-Picture-887.png -images/20100321/1269206238486-dumpfm-jeanette-Picture-587.png -images/20100319/1269032046474-dumpfm-frankhats-cleangetit.png -images/20100320/1269058791399-dumpfm-ryder-idontgetitmoot.png +images/20100321/1269206238486-humpfm-jeanette-Picture-587.png +images/20100319/1269032046474-humpfm-frankhats-cleangetit.png +images/20100320/1269058791399-humpfm-ryder-idontgetitmoot.png images/1266701700856-Picture-4.png images/1266701093659-Picture-1.png images/1266097216406-1266097108677-notgoingtohappen.gif @@ -107,32 +106,30 @@ images/1266193548181-Zlyjt4zgqj9E3UV-S-iK1Q.jpg images/1266234113086-1266190462995_ff567.jpg images/1267361752501-Screen-shot-2010-02-21-at-3.58.28-AM.png images/1268281646619-Screen-shot-2010-03-10-at-8.26.41-PM.png -images/20100312/1268384300040-dumpfm-timb-jertronic.morph.gif -images/20100312/1268387819860-dumpfm-timb-ryder.upsidedown.morph.gif -images/20100319/1268990790344-dumpfm-frankhats-dumphone.gif +images/20100312/1268384300040-humpfm-timb-jertronic.morph.gif +images/20100312/1268387819860-humpfm-timb-ryder.upsidedown.morph.gif images/1268284173872-Screen-shot-2010-03-10-at-9.07.49-PM.png images/1268260418722-notgoingtohappen.gif images/1268262004981-Picture-353.png images/1266361889102-Picture-2.png images/1266273927836-whenwilligetit.gif images/1268096149665-jnetdontgetit.gif -images/20100319/1268990254325-dumpfm-ryder-Picture-1933.png +images/20100319/1268990254325-humpfm-ryder-Picture-1933.png images/1267589980822-1266182446764-dumo.jpeg -images/1266727251964-dump.fm.webcam.test.jpg +images/1266727251964-hump.fm.webcam.test.jpg images/1268160453226-Picture-5.png images/1266195597779-dontgetHDA.gif images/1266196112086-dont.gif images/1267760115236-dont.png images/1267778681746-Picture-223.png -images/20100311/1268344117021-dumpfm-timb-j.morph.gif -images/20100319/1268989639997-dumpfm-ryder-Picture-1930.png -images/20100319/1268989484800-dumpfm-jeanette-Picture-538.png -images/20100319/1268989172037-dumpfm-mwellsie-guy.gif -images/20100319/1268988777710-dumpfm-ryder-FINALLYIGETIT.jpg -images/20100319/1268988131400-dumpfm-frankhats-dunmom.jpg -images/20100318/1268894764628-dumpfm-frankhats-Untitled-3.png +images/20100311/1268344117021-humpfm-timb-j.morph.gif +images/20100319/1268989639997-humpfm-ryder-Picture-1930.png +images/20100319/1268989484800-humpfm-jeanette-Picture-538.png +images/20100319/1268989172037-humpfm-mwellsie-guy.gif +images/20100319/1268988777710-humpfm-ryder-FINALLYIGETIT.jpg +images/20100319/1268988131400-humpfm-frankhats-dunmom.jpg +images/20100318/1268894764628-humpfm-frankhats-Untitled-3.png images/1266825851804-Picture-1233.png -images/20100317/1268813694153-dumpfm-ryder-gatesdump.jpg images/1267294242665-JNETPIZZA.gif images/1266448158853-idontgetitbot.png images/1267588831314-Picture-1508.png diff --git a/scripts/logreport.py b/scripts/logreport.py index 3e6dbaf..fb7626a 100755 --- a/scripts/logreport.py +++ b/scripts/logreport.py @@ -104,7 +104,7 @@ def build_js_array(pairs, max_val=None): def summarize_window(ts, window, slow_threshold): reqs = 0.0 - dumps = 0.0 + humps = 0.0 images = 0.0 tags = 0.0 chat_users = 0.0 @@ -116,7 +116,7 @@ def summarize_window(ts, window, slow_threshold): if group in ('Images', 'Avatars'): images += 1 elif group in ('Post', 'Upload'): - dumps += 1 + humps += 1 elif group in ('TagAdd', 'TagRemove'): tags += 1 elif group == 'ChatUpdate': @@ -127,7 +127,7 @@ def summarize_window(ts, window, slow_threshold): times = [r['time'] for r in window] return { 'ts': ts, 'reqs': reqs, - 'dumps': dumps, + 'humps': humps, 'images': images, 'tags': tags, 'chat_users': chat_users, @@ -140,14 +140,14 @@ def summarize_window(ts, window, slow_threshold): ReportSetLimit = 100 -class DumpReport(object): +class HumpReport(object): def __init__(self, interval, slow_thresh=SlowReqThreshold): self.windows = [] self.interval = interval self.bytes_sent = 0 self.requests = 0 - self.dumps = 0 + self.humps = 0 self.slow_requests = 0 self.new_users = 0 self.unclassified_urls = set() @@ -191,7 +191,7 @@ class DumpReport(object): group = record['urlgroup'] if group in ('Post', 'Upload'): - self.dumps += 1 + self.humps += 1 elif group == 'Register': self.new_users += 1 @@ -250,8 +250,8 @@ class DumpReport(object): build_js_array((r['ts'], r['reqs'] / self.interval.seconds) for r in self.windows)) writer.graph('Images/sec', 'images_per_sec', build_js_array((r['ts'], r['images'] / self.interval.seconds) for r in self.windows)) - writer.graph('Dumps/sec', 'dumps_per_sec', - build_js_array((r['ts'], r['dumps'] / self.interval.seconds) for r in self.windows)) + writer.graph('Humps/sec', 'humps_per_sec', + build_js_array((r['ts'], r['humps'] / self.interval.seconds) for r in self.windows)) writer.graph('Tags/sec', 'tags_per_sec', build_js_array((r['ts'], r['tags'] / self.interval.seconds) for r in self.windows)) writer.graph('Chat users/sec', 'users_per_sec', @@ -287,7 +287,7 @@ class DumpReport(object): self._output_errors_and_unclassified(writer) def write_report(self, outdir): - page_title = 'Dump Report: %s to %s' % (format_time(self.start_time), + page_title = 'Hump Report: %s to %s' % (format_time(self.start_time), format_time(self.end_time)) filename = self.start_time.strftime('%Y-%m-%d__%H-%M.html') path = os.path.join(outdir, filename) @@ -325,7 +325,7 @@ def yield_log_windows(raw_iter, interval, parse_func, timekey): yield start_time + (interval * i), list(g) -# Dump log parsing +# Hump log parsing def parse_time(s): # Parse times such as 02/Jul/2010:22:21:25 -0400 @@ -393,7 +393,7 @@ def classify_url(method, url): elif url == '/upload/message': return 'Upload' elif url.startswith('/p/'): - return 'ViewDump' + return 'ViewHump' else: return 'Other' @@ -431,7 +431,7 @@ LineFormat = [('addr', str), ('refer', str), ('agent', str)] -def parse_dump_log_entry(i, l, error_func=None): +def parse_hump_log_entry(i, l, error_func=None): comps = [c.strip() for c in l.split('|')] if len(comps) != len(LineFormat): if error_func: @@ -456,14 +456,14 @@ def default_error_handler(i, line, err): # print "line %s: %s" % (i, err) pass -default_line_parser = functools.partial(parse_dump_log_entry, +default_line_parser = functools.partial(parse_hump_log_entry, error_func=default_error_handler) -def analyze_dump_log(path, +def analyze_hump_log(path, interval=WindowInterval, line_parser=default_line_parser, timekey='ts'): - report = DumpReport(interval) + report = HumpReport(interval) with open(path, 'r') as f: for ts, window in yield_log_windows(f, interval, line_parser, timekey): report.record_window(ts, window) @@ -477,7 +477,7 @@ if __name__ == "__main__": outpath = sys.argv[2] if len(sys.argv) > 2 else None Nickset = fetch_nicks() - report = analyze_dump_log(logpath) + report = analyze_hump_log(logpath) if outpath: report.write_report(outpath) |