From 02a0597e39ed337ba4dc0cccddebb4c98c63b5ee Mon Sep 17 00:00:00 2001 From: dumpfmprod Date: Sun, 3 Oct 2010 21:28:23 -0400 Subject: timb: better hiscore code --- scripts/hiscores.py | 80 +++++++++++++++++++++++++++++++++++++++++ scripts/hiscores/.month.py.swp | Bin 12288 -> 0 bytes scripts/hiscores/day.py | 35 ------------------ scripts/hiscores/month.py | 35 ------------------ scripts/hiscores/week.py | 35 ------------------ 5 files changed, 80 insertions(+), 105 deletions(-) create mode 100644 scripts/hiscores.py delete mode 100644 scripts/hiscores/.month.py.swp delete mode 100644 scripts/hiscores/day.py delete mode 100644 scripts/hiscores/month.py delete mode 100644 scripts/hiscores/week.py (limited to 'scripts') diff --git a/scripts/hiscores.py b/scripts/hiscores.py new file mode 100644 index 0000000..e8a5ed0 --- /dev/null +++ b/scripts/hiscores.py @@ -0,0 +1,80 @@ +# this needs python 3 due to py-postgresql... + +import re +import sys +import postgresql +import redis + +db = postgresql.open("pq://postgres:root@localhost/dumpfm") +db.execute("SET CLIENT_ENCODING to 'UNICODE'") + +r = redis.Redis("localhost") + +key_prefix = "hiscore:" +hiscore_len = 40 + +config = { + "day": {"days": 1, "amt": hiscore_len}, + "week": {"days": 7, "amt": hiscore_len}, + "month": {"days": 30, "amt": hiscore_len}, + "all": {"days": 0, "amt": hiscore_len} +} + +def fetch_favs(days): + statement = """SELECT message_id + FROM tags + WHERE + LOWER(tag) = 'favorite' """ + if days > 0: + statement = statement + "AND created_on > (now() - INTERVAL '" + str(days) + " day')" + + ps = db.prepare(statement) + return ps.chunks +# using "chunks()" is apparently most efficient way to stream tons of results + +def add_favs_to_redis(key, chunk): + counter = 0 + for rowset in chunks(): + for row in rowset: + message_id = row[0] + r.zincrby(key, message_id, 1) + counter += 1 + if counter % 1000 == 0: + print("processing row", counter) + +def rm_low_scores(key, amt): + amt = (amt + 1) * -1 + r.zremrangebyrank(key, 0, amt) + +def switch_keys(key, keyfinal): + r.rename(key, keyfinal) + +if __name__ == "__main__": + error = False + + if not len(sys.argv) == 2: + error = True + else: + period = sys.argv[1] + if period not in config: + error = True + else: + days = config[period]['days'] + amt = config[period]['amt'] + + if error: + print('usage: python3.1 hiscores.py period') + print('where period is one of:') + for period in config: + print(period) + print('this script adds message ids to redis for the highest scoring posts over a period.') + sys.exit(1) + + keyfinal = key_prefix + period + key = keyfinal + ":temp" + # write to key and then overwrite keyfinal when complete + + chunks = fetch_favs(days) + add_favs_to_redis(key, chunks) + rm_low_scores(key, amt) + switch_keys(key, keyfinal) diff --git a/scripts/hiscores/.month.py.swp b/scripts/hiscores/.month.py.swp deleted file mode 100644 index 4293f41..0000000 Binary files a/scripts/hiscores/.month.py.swp and /dev/null differ diff --git a/scripts/hiscores/day.py b/scripts/hiscores/day.py deleted file mode 100644 index d59ecdf..0000000 --- a/scripts/hiscores/day.py +++ /dev/null @@ -1,35 +0,0 @@ -# this needs python 3 - -import re -import sys -import postgresql -from urllib.parse import urlparse -import redis - -db = postgresql.open("pq://postgres:root@localhost/dumpfm") -db.execute("SET CLIENT_ENCODING to 'UNICODE'") - -r = redis.Redis("localhost") - -def clear_redis(): - r.delete("hiscore.day") - -def fetch_favs(): - ps = db.prepare("SELECT user_id, message_id, created_on FROM tags WHERE created_on > (now() - INTERVAL '1 day') AND LOWER(tag) = 'favorite'") - rows = ps() - return rows - -def add_favs_to_redis(favs): - for fav in favs: - user_id = fav[0] - message_id = fav[1] - r.zincrby("hiscore.day", message_id, 1) - -def rm_low_scores(): - r.zremrangebyscore("hiscore.day", 0, 5) - -if __name__ == "__main__": - clear_redis() - favs = fetch_favs() - add_favs_to_redis(favs) - rm_low_scores() diff --git a/scripts/hiscores/month.py b/scripts/hiscores/month.py deleted file mode 100644 index 616cccc..0000000 --- a/scripts/hiscores/month.py +++ /dev/null @@ -1,35 +0,0 @@ -# this needs python 3 - -import re -import sys -import postgresql -from urllib.parse import urlparse -import redis - -db = postgresql.open("pq://postgres:root@localhost/dumpfm") -db.execute("SET CLIENT_ENCODING to 'UNICODE'") - -r = redis.Redis("localhost") - -def clear_redis(): - r.delete("hiscore.day") - -def fetch_favs(): - ps = db.prepare("SELECT user_id, message_id, created_on FROM tags WHERE created_on > (now() - INTERVAL '30 day') AND LOWER(tag) = 'favorite'") - rows = ps() - return rows - -def add_favs_to_redis(favs): - for fav in favs: - user_id = fav[0] - message_id = fav[1] - r.zincrby("hiscore.month", message_id, 1) - -def rm_low_scores(): - r.zremrangebyscore("hiscore.month", 0, 5) - -if __name__ == "__main__": - clear_redis() - favs = fetch_favs() - add_favs_to_redis(favs) - rm_low_scores() diff --git a/scripts/hiscores/week.py b/scripts/hiscores/week.py deleted file mode 100644 index 3853abd..0000000 --- a/scripts/hiscores/week.py +++ /dev/null @@ -1,35 +0,0 @@ -# this needs python 3 - -import re -import sys -import postgresql -from urllib.parse import urlparse -import redis - -db = postgresql.open("pq://postgres:root@localhost/dumpfm") -db.execute("SET CLIENT_ENCODING to 'UNICODE'") - -r = redis.Redis("localhost") - -def clear_redis(): - r.delete("hiscore.day") - -def fetch_favs(): - ps = db.prepare("SELECT user_id, message_id, created_on FROM tags WHERE created_on > (now() - INTERVAL '7 day') AND LOWER(tag) = 'favorite'") - rows = ps() - return rows - -def add_favs_to_redis(favs): - for fav in favs: - user_id = fav[0] - message_id = fav[1] - r.zincrby("hiscore.week", message_id, 1) - -def rm_low_scores(): - r.zremrangebyscore("hiscore.week", 0, 5) - -if __name__ == "__main__": - clear_redis() - favs = fetch_favs() - add_favs_to_redis(favs) - rm_low_scores() -- cgit v1.2.3-70-g09d2