From 6b8215267bc5af44dfab4097992a31b35513b102 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Sun, 14 Nov 2010 12:55:55 -0800 Subject: Add redisload.clj for caching popular pages --- src/redisload.clj | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 src/redisload.clj (limited to 'src/redisload.clj') diff --git a/src/redisload.clj b/src/redisload.clj new file mode 100644 index 0000000..fdc2664 --- /dev/null +++ b/src/redisload.clj @@ -0,0 +1,72 @@ +(ns redisload + (:use clojure.contrib.sql + config + datalayer + utils) + (:require redis)) + +(defn redis-days [n] + (* 24 60 60)) + +(def tag-query " +SELECT + u.nick as author, + u.user_id as author_id, + m.message_id as message_id, + m.content as message_content, + m.is_image as is_image, + m.created_on as message_ts, + r.key as room, + t.created_on as tagged_on, + u2.nick as tagger, + u2.user_id as tagger_id +FROM users u, messages m, rooms r, tags t, users u2 +WHERE + u.user_id = m.user_id AND + m.message_id = t.message_id AND + r.room_id = m.room_id AND + u2.user_id = t.user_id +") + +(def tag-counter (ref 0)) + +(defn stream-tags [fs] + (with-connection *db* + (with-query-results rs [tag-query] + (doseq [r rs] + (dosync + (ref-set tag-counter (inc @tag-counter))) + (doseq [f fs] + (f r)))))) + + +(def popular-map (ref {})) + +(defn update-popular [r] + (if (:is_image r) + (dosync + (let [user-map (or (get @popular-map (:author r)) {}) + msg-cnt (or (get user-map (:message_id r)) 0)] + (alter popular-map assoc (:author r) + (assoc user-map (:message_id r) (inc msg-cnt))))))) + +(defn transmit-popular [] + (doseq [[nick msgs] @popular-map] + (let [sorted-msgs (sort #(>= (second %1) (second %2)) msgs) + userkey (str "popular:" nick)] + (redis/atomically + (redis/del key) + (doseq [[msg-id score] (take (* num-popular-dumps 2) + sorted-msgs)] + (redis/zadd userkey score msg-id)) + ;; (redis/expire userkey (redis-days 2))))) -- need to schedule auto loader + (println "cached popular data for" (count @popular-map) "users")) + +(println "streaming tags") +(stream-tags [update-popular]) +(println (format "processed %s tags" @tag-counter)) + + +(redis/with-server redis-server + (transmit-popular)) + -- cgit v1.2.3-70-g09d2 From 9b5fb453d7fbe79918fe012badbf6a515092cb48 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Sun, 14 Nov 2010 13:26:44 -0800 Subject: Fixed redisload typo --- src/redisload.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/redisload.clj') diff --git a/src/redisload.clj b/src/redisload.clj index fdc2664..9e33039 100644 --- a/src/redisload.clj +++ b/src/redisload.clj @@ -58,7 +58,7 @@ WHERE (redis/del key) (doseq [[msg-id score] (take (* num-popular-dumps 2) sorted-msgs)] - (redis/zadd userkey score msg-id)) + (redis/zadd userkey score msg-id))))) ;; (redis/expire userkey (redis-days 2))))) -- need to schedule auto loader (println "cached popular data for" (count @popular-map) "users")) -- cgit v1.2.3-70-g09d2 From 52090faa3ca95bbcb1bcea638834f72cf3d93e87 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Fri, 19 Nov 2010 00:55:06 -0500 Subject: redisload.clj: hide admin only --- src/redisload.clj | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src/redisload.clj') diff --git a/src/redisload.clj b/src/redisload.clj index 9e33039..0e34e2c 100644 --- a/src/redisload.clj +++ b/src/redisload.clj @@ -17,6 +17,7 @@ SELECT m.is_image as is_image, m.created_on as message_ts, r.key as room, + r.admin_only as admin_only, t.created_on as tagged_on, u2.nick as tagger, u2.user_id as tagger_id @@ -43,7 +44,7 @@ WHERE (def popular-map (ref {})) (defn update-popular [r] - (if (:is_image r) + (if (and (:is_image r) (not (:admin_only r))) (dosync (let [user-map (or (get @popular-map (:author r)) {}) msg-cnt (or (get user-map (:message_id r)) 0)] @@ -59,7 +60,7 @@ WHERE (doseq [[msg-id score] (take (* num-popular-dumps 2) sorted-msgs)] (redis/zadd userkey score msg-id))))) - ;; (redis/expire userkey (redis-days 2))))) -- need to schedule auto loader + ;; (redis/expire userkey (redis-days 2))))) -- TODO: schedule auto loader (println "cached popular data for" (count @popular-map) "users")) (println "streaming tags") -- cgit v1.2.3-70-g09d2 From 2b7162504314d4207df4c462d3cad12ad144b7c0 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Tue, 23 Nov 2010 01:23:57 -0500 Subject: Add directory/hall to redis loader --- src/redisload.clj | 45 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 9 deletions(-) (limited to 'src/redisload.clj') diff --git a/src/redisload.clj b/src/redisload.clj index 0e34e2c..07c1275 100644 --- a/src/redisload.clj +++ b/src/redisload.clj @@ -1,5 +1,6 @@ (ns redisload (:use clojure.contrib.sql + clojure.contrib.str-utils config datalayer utils) @@ -26,7 +27,8 @@ WHERE u.user_id = m.user_id AND m.message_id = t.message_id AND r.room_id = m.room_id AND - u2.user_id = t.user_id + u2.user_id = t.user_id AND + r.admin_only = false ") (def tag-counter (ref 0)) @@ -35,21 +37,30 @@ WHERE (with-connection *db* (with-query-results rs [tag-query] (doseq [r rs] + (if (:admin_only r) + (println r)) (dosync (ref-set tag-counter (inc @tag-counter))) (doseq [f fs] (f r)))))) - +(def hall-map (ref {})) (def popular-map (ref {})) +(def score-map (ref {})) (defn update-popular [r] (if (and (:is_image r) (not (:admin_only r))) (dosync - (let [user-map (or (get @popular-map (:author r)) {}) - msg-cnt (or (get user-map (:message_id r)) 0)] - (alter popular-map assoc (:author r) - (assoc user-map (:message_id r) (inc msg-cnt))))))) + (let [author (:author r) + msg-id (:message_id r) + user-map (get @popular-map author {}) + msg-cnt (get user-map msg-id 0) + hall-cnt (get @hall-map msg-id 0) + usr-cnt (get @score-map author 0)] + (alter hall-map assoc msg-id (inc hall-cnt)) + (alter score-map assoc author (inc usr-cnt)) + (alter popular-map assoc author + (assoc user-map msg-id (inc msg-cnt))))))) (defn transmit-popular [] (doseq [[nick msgs] @popular-map] @@ -60,14 +71,30 @@ WHERE (doseq [[msg-id score] (take (* num-popular-dumps 2) sorted-msgs)] (redis/zadd userkey score msg-id))))) - ;; (redis/expire userkey (redis-days 2))))) -- TODO: schedule auto loader (println "cached popular data for" (count @popular-map) "users")) +(defn transmit-favscores [] + (redis/atomically + (redis/del "favscores") + (doseq [[nick score] @score-map] + (redis/zadd "favscores" score (lower-case nick)))) + (println "cached favscores for " (count @score-map) "users")) + +(defn transmit-hall [] + (let [scores (take (* 2 num-hall-dumps) + (sort #(>= (second %1) (second %2)) @hall-map))] + (redis/atomically + (redis/del "hall") + (doseq [[msg-id score] scores] + (redis/zadd "hall" score msg-id))))) + + (println "streaming tags") (stream-tags [update-popular]) (println (format "processed %s tags" @tag-counter)) - (redis/with-server redis-server - (transmit-popular)) + (transmit-favscores) + (transmit-popular) + (transmit-hall)) -- cgit v1.2.3-70-g09d2 From ad83cbd56471a628cf4cd657bdea962f4f323973 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Tue, 23 Nov 2010 01:40:43 -0500 Subject: Count txt messages for favscore --- src/redisload.clj | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'src/redisload.clj') diff --git a/src/redisload.clj b/src/redisload.clj index 07c1275..31a25f0 100644 --- a/src/redisload.clj +++ b/src/redisload.clj @@ -49,16 +49,16 @@ WHERE (def score-map (ref {})) (defn update-popular [r] - (if (and (:is_image r) (not (:admin_only r))) - (dosync - (let [author (:author r) - msg-id (:message_id r) - user-map (get @popular-map author {}) - msg-cnt (get user-map msg-id 0) - hall-cnt (get @hall-map msg-id 0) - usr-cnt (get @score-map author 0)] - (alter hall-map assoc msg-id (inc hall-cnt)) - (alter score-map assoc author (inc usr-cnt)) + (dosync + (let [author (:author r) + msg-id (:message_id r) + user-map (get @popular-map author {}) + msg-cnt (get user-map msg-id 0) + hall-cnt (get @hall-map msg-id 0) + usr-cnt (get @score-map author 0)] + (alter score-map assoc author (inc usr-cnt)) + (alter hall-map assoc msg-id (inc hall-cnt)) + (if (:is_image r) (alter popular-map assoc author (assoc user-map msg-id (inc msg-cnt))))))) -- cgit v1.2.3-70-g09d2