1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
|
# this needs python 3 due to py-postgresql...
import re
import sys
import postgresql
import redis
db = postgresql.open("pq://postgres:root@localhost/dumpfm")
db.execute("SET CLIENT_ENCODING to 'UNICODE'")
r = redis.Redis("localhost")
key_prefix = "hiscore:"
hiscore_len = 40
config = {
"day": {"days": 1, "amt": hiscore_len},
"week": {"days": 7, "amt": hiscore_len},
"month": {"days": 30, "amt": hiscore_len},
"all": {"days": 0, "amt": hiscore_len}
}
def fetch_favs(days):
statement = """SELECT message_id
FROM tags
WHERE
LOWER(tag) = 'favorite' """
if days > 0:
statement = statement + "AND created_on > (now() - INTERVAL '" + str(days) + " day')"
ps = db.prepare(statement)
return ps.chunks
# using "chunks()" is apparently most efficient way to stream tons of results
def add_favs_to_redis(key, chunk):
counter = 0
for rowset in chunks():
for row in rowset:
message_id = row[0]
r.zincrby(key, message_id, 1)
counter += 1
if counter % 1000 == 0:
print("processing row", counter)
def rm_low_scores(key, amt):
amt = (amt + 1) * -1
r.zremrangebyrank(key, 0, amt)
def switch_keys(key, keyfinal):
r.rename(key, keyfinal)
if __name__ == "__main__":
error = False
if not len(sys.argv) == 2:
error = True
else:
period = sys.argv[1]
if period not in config:
error = True
else:
days = config[period]['days']
amt = config[period]['amt']
if error:
print('usage: python3.1 hiscores.py period')
print('where period is one of:')
for period in config:
print(period)
print('this script adds message ids to redis for the highest scoring posts over a period.')
sys.exit(1)
keyfinal = key_prefix + period
key = keyfinal + ":temp"
# write to key and then overwrite keyfinal when complete
chunks = fetch_favs(days)
add_favs_to_redis(key, chunks)
rm_low_scores(key, amt)
switch_keys(key, keyfinal)
|
