From e5f01c9ad63b4547900a488c452523a3f88344c5 Mon Sep 17 00:00:00 2001 From: dumpfmprod Date: Sun, 3 Oct 2010 09:08:19 -0400 Subject: timb: testing using redis to store a sorted set of post favcounts --- scripts/hiscores/.month.py.swp | Bin 0 -> 12288 bytes scripts/hiscores/day.py | 35 +++++++++++++++++++++++++ scripts/hiscores/month.py | 35 +++++++++++++++++++++++++ scripts/hiscores/week.py | 35 +++++++++++++++++++++++++ src/site.clj | 29 +++++++++++++++++++++ src/tags.clj | 6 +++-- template/hiscore_test.st | 58 +++++++++++++++++++++++++++++++++++++++++ template/share_buttons.st | 5 +++- 8 files changed, 200 insertions(+), 3 deletions(-) create mode 100644 scripts/hiscores/.month.py.swp create mode 100644 scripts/hiscores/day.py create mode 100644 scripts/hiscores/month.py create mode 100644 scripts/hiscores/week.py create mode 100644 template/hiscore_test.st diff --git a/scripts/hiscores/.month.py.swp b/scripts/hiscores/.month.py.swp new file mode 100644 index 0000000..4293f41 Binary files /dev/null and b/scripts/hiscores/.month.py.swp differ diff --git a/scripts/hiscores/day.py b/scripts/hiscores/day.py new file mode 100644 index 0000000..d59ecdf --- /dev/null +++ b/scripts/hiscores/day.py @@ -0,0 +1,35 @@ +# this needs python 3 + +import re +import sys +import postgresql +from urllib.parse import urlparse +import redis + +db = postgresql.open("pq://postgres:root@localhost/dumpfm") +db.execute("SET CLIENT_ENCODING to 'UNICODE'") + +r = redis.Redis("localhost") + +def clear_redis(): + r.delete("hiscore.day") + +def fetch_favs(): + ps = db.prepare("SELECT user_id, message_id, created_on FROM tags WHERE created_on > (now() - INTERVAL '1 day') AND LOWER(tag) = 'favorite'") + rows = ps() + return rows + +def add_favs_to_redis(favs): + for fav in favs: + user_id = fav[0] + message_id = fav[1] + r.zincrby("hiscore.day", message_id, 1) + +def rm_low_scores(): + r.zremrangebyscore("hiscore.day", 0, 5) + +if __name__ == "__main__": + clear_redis() + favs = fetch_favs() + add_favs_to_redis(favs) + rm_low_scores() diff --git a/scripts/hiscores/month.py b/scripts/hiscores/month.py new file mode 100644 index 0000000..616cccc --- /dev/null +++ b/scripts/hiscores/month.py @@ -0,0 +1,35 @@ +# this needs python 3 + +import re +import sys +import postgresql +from urllib.parse import urlparse +import redis + +db = postgresql.open("pq://postgres:root@localhost/dumpfm") +db.execute("SET CLIENT_ENCODING to 'UNICODE'") + +r = redis.Redis("localhost") + +def clear_redis(): + r.delete("hiscore.day") + +def fetch_favs(): + ps = db.prepare("SELECT user_id, message_id, created_on FROM tags WHERE created_on > (now() - INTERVAL '30 day') AND LOWER(tag) = 'favorite'") + rows = ps() + return rows + +def add_favs_to_redis(favs): + for fav in favs: + user_id = fav[0] + message_id = fav[1] + r.zincrby("hiscore.month", message_id, 1) + +def rm_low_scores(): + r.zremrangebyscore("hiscore.month", 0, 5) + +if __name__ == "__main__": + clear_redis() + favs = fetch_favs() + add_favs_to_redis(favs) + rm_low_scores() diff --git a/scripts/hiscores/week.py b/scripts/hiscores/week.py new file mode 100644 index 0000000..3853abd --- /dev/null +++ b/scripts/hiscores/week.py @@ -0,0 +1,35 @@ +# this needs python 3 + +import re +import sys +import postgresql +from urllib.parse import urlparse +import redis + +db = postgresql.open("pq://postgres:root@localhost/dumpfm") +db.execute("SET CLIENT_ENCODING to 'UNICODE'") + +r = redis.Redis("localhost") + +def clear_redis(): + r.delete("hiscore.day") + +def fetch_favs(): + ps = db.prepare("SELECT user_id, message_id, created_on FROM tags WHERE created_on > (now() - INTERVAL '7 day') AND LOWER(tag) = 'favorite'") + rows = ps() + return rows + +def add_favs_to_redis(favs): + for fav in favs: + user_id = fav[0] + message_id = fav[1] + r.zincrby("hiscore.week", message_id, 1) + +def rm_low_scores(): + r.zremrangebyscore("hiscore.week", 0, 5) + +if __name__ == "__main__": + clear_redis() + favs = fetch_favs() + add_favs_to_redis(favs) + rm_low_scores() diff --git a/src/site.clj b/src/site.clj index c25a18d..c9b484b 100644 --- a/src/site.clj +++ b/src/site.clj @@ -848,6 +848,28 @@ WHERE u.user_id = ANY(?)" (log session (lookup-room room-key) offset params) (resp-error "UNKNOWN_ROOM")))) +;; Hiscore test... redis test... + +(defn redis-ids-test [period] + (let [reddis-server {:host "127.0.0.1" :port 6379 :db 0} + ids (redis/with-server reddis-server + (redis/zrevrange (str "hiscore." period) 0 20)) + ids (map maybe-parse-int ids)] + ids)) + +(defn hiscore-test [session params period] + (let [st (fetch-template "hiscore_test" session) + dumps (tags/fetch-dumps-by-ids (redis-ids-test period)) + dumps (map tags/add-favorited-flag dumps (repeat session)) + dumps (map tags/add-fav-count dumps) + dumps (reverse (sort-by :favcount dumps)) + dumps (map tags/remove-tags-for-output dumps) + dumps (map process-message-for-output dumps)] + (.setAttribute st "dumps" dumps) + (.toString st))) + + + ;; Altars (defn altar-log [session params] @@ -1304,6 +1326,13 @@ WHERE u.user_id = ANY(?)" (altar-log session params) (redirect-to (str "/" (params :id) "/altars")))) ;; redirect /altars/timb to /timb/altars + ;; testing + (GET "/test/hiscores" (hiscore-test session params "week")) + (GET "/test/hiscores/week" (hiscore-test session params "week")) + (GET "/test/hiscores/month" (hiscore-test session params "month")) + + + ;; Events (GET "/event" (current-event session)) diff --git a/src/tags.clj b/src/tags.clj index 01414e9..835591a 100644 --- a/src/tags.clj +++ b/src/tags.clj @@ -64,6 +64,9 @@ (assoc row :favorited true) row)) +(defn add-fav-count [row] + (assoc row :favcount (count ((row :tags) "favorite")))) + (def user-fav-query " SELECT v FROM UNNEST(?) as v @@ -259,8 +262,7 @@ WHERE EXISTS WHERE m.message_id IN (" (str-join ", " (take num-messages (repeat "?"))) ") " " AND m.user_id = u.user_id - AND r.room_id = m.room_id - ORDER BY m.message_id DESC")) + AND r.room_id = m.room_id")) (defnk fetch-dumps-by-tag-query [:image-only true :message-user-id false :tag-user-id false :with-tags true :include-vip false] (str diff --git a/template/hiscore_test.st b/template/hiscore_test.st new file mode 100644 index 0000000..e700bb0 --- /dev/null +++ b/template/hiscore_test.st @@ -0,0 +1,58 @@ + + + dump.fm log +$head()$ + + + +$banner()$ +
+
+
+

most fav'd dumps this week


+
+
+ + $if(dumps)$ +
+ $dumps: { d | $log_dump(dump=d)$ }$ +
+ $if(json_tags)$ + + $endif$ + $else$ + No dumps! + $endif$ +
+ + +
+
+ + + +
+
+ $footer()$ +
+
+
+ + + diff --git a/template/share_buttons.st b/template/share_buttons.st index 882ec07..7cb874f 100644 --- a/template/share_buttons.st +++ b/template/share_buttons.st @@ -11,4 +11,7 @@ $else$ $endif$ - \ No newline at end of file +$if(dump.favcount)$ + $dump.favcount$ +$endif$ + -- cgit v1.2.3-70-g09d2