diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/fix_fav_counts.py | 99 | ||||
| -rw-r--r-- | scripts/idgi/filenames.list.txt | 165 | ||||
| -rwxr-xr-x | scripts/idgi/rename.sh | 12 | ||||
| -rw-r--r-- | scripts/mia/iwant.flv | bin | 0 -> 986466 bytes | |||
| -rwxr-xr-x | scripts/sort_faves.sh | 20 |
5 files changed, 296 insertions, 0 deletions
diff --git a/scripts/fix_fav_counts.py b/scripts/fix_fav_counts.py new file mode 100644 index 0000000..00e8d0f --- /dev/null +++ b/scripts/fix_fav_counts.py @@ -0,0 +1,99 @@ +""" + this needs python 3 due to py-postgresql... + + before running this command, please run the following SQL, which tallies the faves per post: + + COPY (SELECT messages.user_id, tags.message_id, COUNT(tags.message_id) AS mycount, TO_CHAR(messages.created_on, 'YYYYMMDD') + FROM tags, messages + WHERE tags.message_id = messages.message_id AND tags.tag = 'favorite' + GROUP BY tags.message_id, messages.user_id, messages.created_on) + TO '/tmp/fav_counts2.csv' WITH CSV; + + then run ./sort_faves.sh which will pre-sort the data for this script. +""" + +import re +import sys +import postgresql +import redis +import csv + +db = postgresql.open("pq://postgres:root@localhost/dumpfm") +db.execute("SET CLIENT_ENCODING to 'UNICODE'") + +r = redis.Redis("192.168.156.111") + +def fetch_users(): + statement = """SELECT user_id, nick FROM users""" + + ps = db.prepare(statement) + return ps.chunks + +# by_date.csv by_user.csv counts_sorted.csv hall.csv +# field order: user_id, message_id, score, date + +def load_faves_by_user(): + nicks = load_nicks() + counter = 0 + user_counter = 0 + score = 0 + user_id = 0 + key = "" + with open('faves/by_user.csv', 'r') as csvfile: + reader = csv.reader(csvfile) + for row in reader: + if row[0] != user_id: + if score != 0: + r.zadd("favscores", nicks[user_id], score) + counter = 0 + user_counter += 1 + score = 0 + user_id = row[0] + key = "popular:" + nicks[user_id] + if (user_counter % 1000) == 0: + print(str(user_counter) + " ...") + score += int(row[2]) + if counter > 30: + continue + r.zadd(key, row[1], int(row[2])) + counter += 1 + +def load_faves_by_date(): + date_counter = 0 + counter = 0 + date = "" + key = "" + with open('faves/by_date.csv', 'r') as csvfile: + reader = csv.reader(csvfile) + for row in reader: + if row[3] != date: + counter = 0 + date_counter += 1 + date = row[3] + key = "hall:daily:" + row[3] + if (int(date) % 100) == 1: + print(key) + if counter > 30: + continue + r.zadd(key, row[1], int(row[2])) + counter += 1 + +def load_hall(): + with open('faves/hall.csv', 'r') as csvfile: + reader = csv.reader(csvfile) + for row in reader: + r.zadd('hall', row[1], int(row[2])) + +def load_nicks(): + nicks = {} + chunks = fetch_users() + for rowset in chunks(): + for row in rowset: + nicks[str(row[0])] = row[1] + return nicks + +if __name__ == "__main__": + # load_hall() + # load_faves_by_user() + load_faves_by_date() + diff --git a/scripts/idgi/filenames.list.txt b/scripts/idgi/filenames.list.txt new file mode 100644 index 0000000..a3776f1 --- /dev/null +++ b/scripts/idgi/filenames.list.txt @@ -0,0 +1,165 @@ +images/1266097108677-notgoingtohappen.jpg +images/1266255579876-dontgetit.png +images/20100312/1268412932681-dumpfm-andrej-this-guy-knows.gif +images/20100418/1271584646128-dumpfm-whatevz-ist2_276319-corporate-your-message-here.jpg +images/20100329/1269836545708-dumpfm-ryder-usdancing.gif +images/20100329/1269836638081-dumpfm-ryder-dudedancing.gif +images/20100330/1269924366027-dumpfm-frankhats-bedunno.jpg +images/20100330/1269930997328-dumpfm-ryder-Picture-41.png +images/20100331/1270082757023-dumpfm-jeanette-Picture-3.png +images/20100401/1270100522426-dumpfm-ryder-Picture-90.png +images/20100401/1270179079682-dumpfm-ryder-Picture-114.png +images/20100401/1270179478809-dumpfm-ryder-wolfguy.gif +images/20100402/1270195156779-dumpfm-frankhats-dunliq.jpg +images/20100402/1270196318946-dumpfm-illalli-hothat.jpg +images/20100401/1270151653306-dumpfm-ryder-Picture-108.png +images/20100404/1270371704503-dumpfm-frankhats-crudun.gif +images/20100404/1270436206347-dumpfm-frankhats-dungetilitter.gif +images/20100405/1270449719939-dumpfm-illalli-likemike.jpg +images/20100405/1270451213193-dumpfm-ryder-Picture-187.png +images/20100405/1270453310699-dumpfm-ryder-yruignoring.gif +images/20100405/1270454430952-dumpfm-frankhats-dundora.jpg +images/20100407/1270677806850-dumpfm-frankhats-gildedgetit.jpg +images/20100415/1271321993419-dumpfm-ryder-Picture-452.png +images/20100407/1270697865705-dumpfm-ryder-artpwnd.gif +images/20100407/1270698050400-dumpfm-ryder-dumpfmfresh.gif +images/20100413/1271195512043-dumpfm-noisia-dontgetarm.jpg +images/20100408/1270705138983-dumpfm-ryder-Picture-247.png +images/20100408/1270758039197-dumpfm-ryder-Picture-254.png +images/20100408/1270764920229-dumpfm-ryder-Picture-260.png +images/20100409/1270801759113-dumpfm-jertronic-nogetit.png +images/20100328/1269805908876-dumpfm-ryder-dumplift.gif +images/20100319/1268991384841-dumpfm-jeanette-Picture-540.png +images/20100410/1270872008733-dumpfm-frankhats-Picture-14.png +images/20100410/1270872200318-dumpfm-frankhats-dontcatit.jpg +images/20100410/1270873042199-dumpfm-frankhats-catrip.gif +images/20100410/1270953524877-dumpfm-timb-idgi.cat.gif +images/20100410/1270953524877-dumpfm-timb-idgi.cat.gif +images/20100411/1270962214585-dumpfm-ryder-joindumpfm.png +images/20100408/1270708774547-dumpfm-jertronic-idontevengetyou.gif +images/1267264103873-dontgetit.gif +images/1266377745509-popeandguy.gif +images/1266214389930-stilldontgetit.png +images/20100412/1271118251387-dumpfm-ryder-Picture-381.png +images/20100412/1271118841756-dumpfm-jertronic-idgisplode.gif +images/20100412/1271119643882-dumpfm-ryder-Picture-384.png +images/20100413/1271133664413-dumpfm-ryder-Picture-397.png +images/1267044843246-lady.gif +images/1267043111981-Picture-2.png +images/20100413/1271193181035-dumpfm-jeanette-Picture-1047.png +images/20100413/1271193218714-dumpfm-ryder-sniffycatguymason.gif +images/20100413/1271195890446-dumpfm-mat3i-HIM.gif +images/20100415/1271321054370-dumpfm-foot-Picture-53.png +images/20100413/1271195996111-dumpfm-frankhats-dontgethead.jpg +images/20100415/1271389483032-dumpfm-shaq-retard.jpg +images/20100326/1269583520221-dumpfm-ryder-Picture-2081.png +images/20100416/1271399363348-dumpfm-frankhats-dontfriends.jpg +images/20100417/1271489503232-dumpfm-frankhats-dontgetturdeating.gif +images/20100401/1270177487240-dumpfm-ryder-guyfunny.png +images/20100417/1271497185798-dumpfm-foot-ykW0S.jpg +images/20100417/1271497299338-dumpfm-frankhats-getbag.jpg +images/20100405/1270451213193-dumpfm-ryder-Picture-187.png +images/20100405/1270448791853-dumpfm-ryder-Picture-169.png +images/20100403/1270290052415-dumpfm-ryder-Picture-148.png +images/20100402/1270261868168-dumpfm-foot-idk.jpg +images/20100405/1270451552583-dumpfm-ryder-guybreadhover.gif +images/20100417/1271562875087-dumpfm-jertronic-cry-until-get-it.gif +images/20100418/1271579861201-dumpfm-foot-1271579304078-dumpfm-ryder-Picture-516-1.png +images/20100418/1271579977086-dumpfm-ryder-Picture-521.png +images/20100418/1271580228382-dumpfm-ryder-Picture-523.png +images/20100418/1271581669282-dumpfm-ryder-Picture-525.png +images/20100418/1271584909042-dumpfm-frankhats-hiding.jpg +images/20100418/1271579100256-dumpfm-pheezer-idontgetitfuckyou.gif +images/20100418/1271646064613-dumpfm-frankhats-getjobs.gif +images/20100418/1271582779905-dumpfm-frankhats-dontlook.gif +images/20100418/1271637365874-dumpfm-ryder-Picture-546.png +images/20100418/1271637177773-dumpfm-ryder-Picture-545.png +images/20100418/1271628385535-dumpfm-ryder-Picture-543.png +images/20100419/1271654870347-dumpfm-foot-Picture-44.png +images/20100418/1271618348378-dumpfm-johntransue-1271581782422-dumpfm-foot-Picture-81.gif +images/20100419/1271663567628-dumpfm-frankhats-bikeleader.jpg +images/20100418/1271584609909-dumpfm-ryder-Picture-532.png +images/20100419/1271664504149-dumpfm-foot-24044_1382683481693_1070133119_1141455_4968953_n.jpg +images/20100419/1271664900678-dumpfm-foot-1271664772627-dumpfm-jeanette-Picture-524.png +images/20100419/1271665856679-dumpfm-jeanette-Picture-1203.png +images/20100324/1269407028795-dumpfm-frankhats-duntv.jpg +images/20100323/1269334306336-dumpfm-ryder-Picture-2031.png +images/20100322/1269243170176-dumpfm-ryder-Picture-1992.png +images/1266884561915-Screen-shot-2010-02-22-at-7.22.13-PM.png +images/1266866394012-Picture-1245.png +images/20100321/1269211476225-dumpfm-jeanette-Picture-590.png +images/1266904219513-Picture-1288.png +images/1266205200000-Picture-887.png +images/20100321/1269206238486-dumpfm-jeanette-Picture-587.png +images/20100319/1269032046474-dumpfm-frankhats-cleangetit.png +images/20100320/1269058791399-dumpfm-ryder-idontgetitmoot.png +images/1266701700856-Picture-4.png +images/1266701093659-Picture-1.png +images/1266097216406-1266097108677-notgoingtohappen.gif +images/1266097476626-1266097108677-notgoingtohappen.jpg +images/1266101477066-gallerie.jpg +images/1266102511308-idgi.gif +images/1266190920179-90_w9YlTyhv6V8Qh6Q0G4g.jpg +images/1266191384380-5fb6SywRT029KqZqTe0NNg.jpg +images/1266191673437-FIrq-Tou6Eg_8zrFAmjV8w.jpg +images/1266192442019-LebbaStMljMGyP7gESyGrA.jpg +images/1266193548181-Zlyjt4zgqj9E3UV-S-iK1Q.jpg +images/1266234113086-1266190462995_ff567.jpg +images/1267361752501-Screen-shot-2010-02-21-at-3.58.28-AM.png +images/1268281646619-Screen-shot-2010-03-10-at-8.26.41-PM.png +images/20100312/1268384300040-dumpfm-timb-jertronic.morph.gif +images/20100312/1268387819860-dumpfm-timb-ryder.upsidedown.morph.gif +images/20100319/1268990790344-dumpfm-frankhats-dumphone.gif +images/1268284173872-Screen-shot-2010-03-10-at-9.07.49-PM.png +images/1268260418722-notgoingtohappen.gif +images/1268262004981-Picture-353.png +images/1266361889102-Picture-2.png +images/1266273927836-whenwilligetit.gif +images/1268096149665-jnetdontgetit.gif +images/20100319/1268990254325-dumpfm-ryder-Picture-1933.png +images/1267589980822-1266182446764-dumo.jpeg +images/1266727251964-dump.fm.webcam.test.jpg +images/1268160453226-Picture-5.png +images/1266195597779-dontgetHDA.gif +images/1266196112086-dont.gif +images/1267760115236-dont.png +images/1267778681746-Picture-223.png +images/20100311/1268344117021-dumpfm-timb-j.morph.gif +images/20100319/1268989639997-dumpfm-ryder-Picture-1930.png +images/20100319/1268989484800-dumpfm-jeanette-Picture-538.png +images/20100319/1268989172037-dumpfm-mwellsie-guy.gif +images/20100319/1268988777710-dumpfm-ryder-FINALLYIGETIT.jpg +images/20100319/1268988131400-dumpfm-frankhats-dunmom.jpg +images/20100318/1268894764628-dumpfm-frankhats-Untitled-3.png +images/1266825851804-Picture-1233.png +images/20100317/1268813694153-dumpfm-ryder-gatesdump.jpg +images/1267294242665-JNETPIZZA.gif +images/1266448158853-idontgetitbot.png +images/1267588831314-Picture-1508.png +images/1267320714013-Picture-1443.png +images/1267300344083-idontgetit.jpg +images/1267232179345-idontgetiteyez.gif +images/1267422032393-Picture-1480.png +images/1266604231028-IFONLYIGOTIT2.png +images/1267589971256-Picture-103.png +images/1267591430660-idontgetitvamp.gif +images/1267668839821-idgimouth.gif +images/1267668923792-Picture-7.gif +images/1267704329936-Picture-195.png +images/1266455113960-Picture-3.png +images/1266362179381-Picture-3.png +images/1266273918102-IDGI.gif +images/1266318507160-guycurtains.gif +images/1266361564489-Picture-105.png +images/1266377477267-Picture-6.png +images/1266356768474-Picture-984.png +images/1266446803730-1266314492419-getitandsucks.gif +images/1266532563132-1266532524431-idontgetitparty.jpg +images/1266453562966-guywithhisdog.png +images/1266177701532-idontgetitfacefat.gif +images/1266871960976-idgi.jpg +images/1266726702281-Picture-1193.png +images/1267232593224-1266189511462-Picture-880.png +images/1267259317284-Picture-30.png +images/1266315286811-Picture-953.png +images/1266320375222-Picture-954.png diff --git a/scripts/idgi/rename.sh b/scripts/idgi/rename.sh new file mode 100755 index 0000000..18af639 --- /dev/null +++ b/scripts/idgi/rename.sh @@ -0,0 +1,12 @@ +#!/bin/bash +FILE="filenames.list.txt" +exec < $FILE +while read LINE + do + NEWL=${LINE/%gif/removed.gif} + NEWL=${NEWL/%jpg/removed.jpg} + NEWL=${NEWL/%jpeg/removed.jpeg} + NEWL=${NEWL/%png/removed.png} + CMDLINE="mv /home/dumpfmprod/prod/$LINE /home/dumpfmprod/prod/$NEWL" + $CMDLINE + done diff --git a/scripts/mia/iwant.flv b/scripts/mia/iwant.flv Binary files differnew file mode 100644 index 0000000..8d957c4 --- /dev/null +++ b/scripts/mia/iwant.flv diff --git a/scripts/sort_faves.sh b/scripts/sort_faves.sh new file mode 100755 index 0000000..eb28a39 --- /dev/null +++ b/scripts/sort_faves.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +if [ ! -n "$1" ] +then + echo "Usage: `basename $0` fav_scores.csv" + exit +fi + +mkdir -p faves + +echo "Get the top 50 faves for the Hall of Fame" +sort -t, -k3,3 -rn $1 | head -50 > faves/hall.csv + +echo "Group faves by user for popular pages, total fave scores" +sort -t, -k1,1 -r -k3,3 -n faves/counts_sorted.csv > faves/by_user.csv + +echo "Group faves by date for daily tallies" +sort -t, -k4,4 -r -k3,3 -n faves/counts_sorted.csv > faves/by_date.csv + + |