summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--scripts/hiscores.py80
-rw-r--r--scripts/hiscores/.month.py.swpbin12288 -> 0 bytes
-rw-r--r--scripts/hiscores/day.py35
-rw-r--r--scripts/hiscores/month.py35
-rw-r--r--scripts/hiscores/week.py35
-rw-r--r--src/site.clj7
-rw-r--r--template/hiscore_test.st7
7 files changed, 92 insertions, 107 deletions
diff --git a/scripts/hiscores.py b/scripts/hiscores.py
new file mode 100644
index 0000000..e8a5ed0
--- /dev/null
+++ b/scripts/hiscores.py
@@ -0,0 +1,80 @@
+# this needs python 3 due to py-postgresql...
+
+import re
+import sys
+import postgresql
+import redis
+
+db = postgresql.open("pq://postgres:root@localhost/dumpfm")
+db.execute("SET CLIENT_ENCODING to 'UNICODE'")
+
+r = redis.Redis("localhost")
+
+key_prefix = "hiscore:"
+hiscore_len = 40
+
+config = {
+ "day": {"days": 1, "amt": hiscore_len},
+ "week": {"days": 7, "amt": hiscore_len},
+ "month": {"days": 30, "amt": hiscore_len},
+ "all": {"days": 0, "amt": hiscore_len}
+}
+
+def fetch_favs(days):
+ statement = """SELECT message_id
+ FROM tags
+ WHERE
+ LOWER(tag) = 'favorite' """
+ if days > 0:
+ statement = statement + "AND created_on > (now() - INTERVAL '" + str(days) + " day')"
+
+ ps = db.prepare(statement)
+ return ps.chunks
+# using "chunks()" is apparently most efficient way to stream tons of results
+
+def add_favs_to_redis(key, chunk):
+ counter = 0
+ for rowset in chunks():
+ for row in rowset:
+ message_id = row[0]
+ r.zincrby(key, message_id, 1)
+ counter += 1
+ if counter % 1000 == 0:
+ print("processing row", counter)
+
+def rm_low_scores(key, amt):
+ amt = (amt + 1) * -1
+ r.zremrangebyrank(key, 0, amt)
+
+def switch_keys(key, keyfinal):
+ r.rename(key, keyfinal)
+
+if __name__ == "__main__":
+ error = False
+
+ if not len(sys.argv) == 2:
+ error = True
+ else:
+ period = sys.argv[1]
+ if period not in config:
+ error = True
+ else:
+ days = config[period]['days']
+ amt = config[period]['amt']
+
+ if error:
+ print('usage: python3.1 hiscores.py period')
+ print('where period is one of:')
+ for period in config:
+ print(period)
+ print('this script adds message ids to redis for the highest scoring posts over a period.')
+ sys.exit(1)
+
+ keyfinal = key_prefix + period
+ key = keyfinal + ":temp"
+ # write to key and then overwrite keyfinal when complete
+
+ chunks = fetch_favs(days)
+ add_favs_to_redis(key, chunks)
+ rm_low_scores(key, amt)
+ switch_keys(key, keyfinal)
diff --git a/scripts/hiscores/.month.py.swp b/scripts/hiscores/.month.py.swp
deleted file mode 100644
index 4293f41..0000000
--- a/scripts/hiscores/.month.py.swp
+++ /dev/null
Binary files differ
diff --git a/scripts/hiscores/day.py b/scripts/hiscores/day.py
deleted file mode 100644
index d59ecdf..0000000
--- a/scripts/hiscores/day.py
+++ /dev/null
@@ -1,35 +0,0 @@
-# this needs python 3
-
-import re
-import sys
-import postgresql
-from urllib.parse import urlparse
-import redis
-
-db = postgresql.open("pq://postgres:root@localhost/dumpfm")
-db.execute("SET CLIENT_ENCODING to 'UNICODE'")
-
-r = redis.Redis("localhost")
-
-def clear_redis():
- r.delete("hiscore.day")
-
-def fetch_favs():
- ps = db.prepare("SELECT user_id, message_id, created_on FROM tags WHERE created_on > (now() - INTERVAL '1 day') AND LOWER(tag) = 'favorite'")
- rows = ps()
- return rows
-
-def add_favs_to_redis(favs):
- for fav in favs:
- user_id = fav[0]
- message_id = fav[1]
- r.zincrby("hiscore.day", message_id, 1)
-
-def rm_low_scores():
- r.zremrangebyscore("hiscore.day", 0, 5)
-
-if __name__ == "__main__":
- clear_redis()
- favs = fetch_favs()
- add_favs_to_redis(favs)
- rm_low_scores()
diff --git a/scripts/hiscores/month.py b/scripts/hiscores/month.py
deleted file mode 100644
index 616cccc..0000000
--- a/scripts/hiscores/month.py
+++ /dev/null
@@ -1,35 +0,0 @@
-# this needs python 3
-
-import re
-import sys
-import postgresql
-from urllib.parse import urlparse
-import redis
-
-db = postgresql.open("pq://postgres:root@localhost/dumpfm")
-db.execute("SET CLIENT_ENCODING to 'UNICODE'")
-
-r = redis.Redis("localhost")
-
-def clear_redis():
- r.delete("hiscore.day")
-
-def fetch_favs():
- ps = db.prepare("SELECT user_id, message_id, created_on FROM tags WHERE created_on > (now() - INTERVAL '30 day') AND LOWER(tag) = 'favorite'")
- rows = ps()
- return rows
-
-def add_favs_to_redis(favs):
- for fav in favs:
- user_id = fav[0]
- message_id = fav[1]
- r.zincrby("hiscore.month", message_id, 1)
-
-def rm_low_scores():
- r.zremrangebyscore("hiscore.month", 0, 5)
-
-if __name__ == "__main__":
- clear_redis()
- favs = fetch_favs()
- add_favs_to_redis(favs)
- rm_low_scores()
diff --git a/scripts/hiscores/week.py b/scripts/hiscores/week.py
deleted file mode 100644
index 3853abd..0000000
--- a/scripts/hiscores/week.py
+++ /dev/null
@@ -1,35 +0,0 @@
-# this needs python 3
-
-import re
-import sys
-import postgresql
-from urllib.parse import urlparse
-import redis
-
-db = postgresql.open("pq://postgres:root@localhost/dumpfm")
-db.execute("SET CLIENT_ENCODING to 'UNICODE'")
-
-r = redis.Redis("localhost")
-
-def clear_redis():
- r.delete("hiscore.day")
-
-def fetch_favs():
- ps = db.prepare("SELECT user_id, message_id, created_on FROM tags WHERE created_on > (now() - INTERVAL '7 day') AND LOWER(tag) = 'favorite'")
- rows = ps()
- return rows
-
-def add_favs_to_redis(favs):
- for fav in favs:
- user_id = fav[0]
- message_id = fav[1]
- r.zincrby("hiscore.week", message_id, 1)
-
-def rm_low_scores():
- r.zremrangebyscore("hiscore.week", 0, 5)
-
-if __name__ == "__main__":
- clear_redis()
- favs = fetch_favs()
- add_favs_to_redis(favs)
- rm_low_scores()
diff --git a/src/site.clj b/src/site.clj
index c9b484b..98fde79 100644
--- a/src/site.clj
+++ b/src/site.clj
@@ -853,7 +853,7 @@ WHERE u.user_id = ANY(?)"
(defn redis-ids-test [period]
(let [reddis-server {:host "127.0.0.1" :port 6379 :db 0}
ids (redis/with-server reddis-server
- (redis/zrevrange (str "hiscore." period) 0 20))
+ (redis/zrevrange (str "hiscore:" period) 0 -1))
ids (map maybe-parse-int ids)]
ids))
@@ -872,6 +872,9 @@ WHERE u.user_id = ANY(?)"
;; Altars
+;; if :nick is in params, will fetch only altars by that nick
+;; next page links look like /altars/message-id and select <= message_id order desc
+;; prev page links look like /altars/-message-id and select > message_id order asc
(defn altar-log [session params]
(let [id (params :id)
nick (params :nick)
@@ -1328,6 +1331,8 @@ WHERE u.user_id = ANY(?)"
;; testing
(GET "/test/hiscores" (hiscore-test session params "week"))
+ (GET "/test/hiscores/alltime" (hiscore-test session params "all"))
+ (GET "/test/hiscores/day" (hiscore-test session params "day"))
(GET "/test/hiscores/week" (hiscore-test session params "week"))
(GET "/test/hiscores/month" (hiscore-test session params "month"))
diff --git a/template/hiscore_test.st b/template/hiscore_test.st
index e700bb0..50e7b9e 100644
--- a/template/hiscore_test.st
+++ b/template/hiscore_test.st
@@ -11,7 +11,12 @@ $banner()$
<div id="content">
<div id="messagePanep">
<div id="userListp">
- <h2> most fav'd dumps this week </h2><br><h3></h3>
+ <h2> most fav'd dumps this
+ <a href="/test/hiscores/day">day</a>,
+ <a href="/test/hiscores/week">week</a>,
+ <a href="/test/hiscores/month">month</a>,
+ <a href="/test/hiscores/all">4ever</a>
+ </h2><br><h3></h3>
</div>
<div id="messageList">
generated by cgit v1.2.3-70-g09d2 (git 2.51.0) at 2025-12-10 10:51:58 +0000