diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/config.clj | 18 | ||||
| -rw-r--r-- | src/redisload.clj | 72 | ||||
| -rw-r--r-- | src/site.clj | 3 | ||||
| -rwxr-xr-x | src/utils.clj | 41 |
4 files changed, 113 insertions, 21 deletions
diff --git a/src/config.clj b/src/config.clj index 3230c72..fdb33ed 100644 --- a/src/config.clj +++ b/src/config.clj @@ -11,10 +11,20 @@ "http://localhost:8080")) (def *cookie-domain* - (if (= *server-user* "timb") - "" - ".dump.fm")) + (if (= *server-user* "dumpfmprod") + ".dump.fm" + "")) + +(def redis-server + (if (= *server-user* "dumpfmprod") + {:host "192.168.156.111" :port 6379 :db 0 } + {:host "127.0.0.1" :port 6379 :db 0 })) (def *root-directory* (System/getProperty "user.dir")) (def *image-directory* "images") -(def *avatar-directory* "avatars")
\ No newline at end of file +(def *avatar-directory* "avatars") + + +;; Numerical constants + +(def num-popular-dumps 40) diff --git a/src/redisload.clj b/src/redisload.clj new file mode 100644 index 0000000..fdc2664 --- /dev/null +++ b/src/redisload.clj @@ -0,0 +1,72 @@ +(ns redisload + (:use clojure.contrib.sql + config + datalayer + utils) + (:require redis)) + +(defn redis-days [n] + (* 24 60 60)) + +(def tag-query " +SELECT + u.nick as author, + u.user_id as author_id, + m.message_id as message_id, + m.content as message_content, + m.is_image as is_image, + m.created_on as message_ts, + r.key as room, + t.created_on as tagged_on, + u2.nick as tagger, + u2.user_id as tagger_id +FROM users u, messages m, rooms r, tags t, users u2 +WHERE + u.user_id = m.user_id AND + m.message_id = t.message_id AND + r.room_id = m.room_id AND + u2.user_id = t.user_id +") + +(def tag-counter (ref 0)) + +(defn stream-tags [fs] + (with-connection *db* + (with-query-results rs [tag-query] + (doseq [r rs] + (dosync + (ref-set tag-counter (inc @tag-counter))) + (doseq [f fs] + (f r)))))) + + +(def popular-map (ref {})) + +(defn update-popular [r] + (if (:is_image r) + (dosync + (let [user-map (or (get @popular-map (:author r)) {}) + msg-cnt (or (get user-map (:message_id r)) 0)] + (alter popular-map assoc (:author r) + (assoc user-map (:message_id r) (inc msg-cnt))))))) + +(defn transmit-popular [] + (doseq [[nick msgs] @popular-map] + (let [sorted-msgs (sort #(>= (second %1) (second %2)) msgs) + userkey (str "popular:" nick)] + (redis/atomically + (redis/del key) + (doseq [[msg-id score] (take (* num-popular-dumps 2) + sorted-msgs)] + (redis/zadd userkey score msg-id)) + ;; (redis/expire userkey (redis-days 2))))) -- need to schedule auto loader + (println "cached popular data for" (count @popular-map) "users")) + +(println "streaming tags") +(stream-tags [update-popular]) +(println (format "processed %s tags" @tag-counter)) + + +(redis/with-server redis-server + (transmit-popular)) + diff --git a/src/site.clj b/src/site.clj index a1f5c08..161f0d5 100644 --- a/src/site.clj +++ b/src/site.clj @@ -1509,6 +1509,3 @@ WHERE u.user_id = ANY(?)" ;(if (not= *server-url* "http://dump.fm") ; (start! random-poster)) - - - diff --git a/src/utils.clj b/src/utils.clj index 1a9e09e..9d7fd3a 100755 --- a/src/utils.clj +++ b/src/utils.clj @@ -18,6 +18,7 @@ clojure.contrib.sql clojure.contrib.def clojure.contrib.duck-streams + clojure.contrib.seq-utils clojure.contrib.str-utils compojure config)) @@ -82,11 +83,12 @@ (declare stringify-and-escape) (defn escape-html-deep [o] - (if (map? o) - (stringify-and-escape o) - (cond (seq? o) (map escape-html-deep o) - (or (true? o) (false? o)) o - :else (escape-html o)))) + (cond (map? o) (stringify-and-escape o) + (vector? o) (map escape-html-deep o) + (seq? o) (map escape-html-deep o) + (true? o) o + (false? o) o + :else (escape-html o))) (defn stringify-and-escape [m] (zipmap (map str* (keys m)) (map escape-html-deep (vals m)))) @@ -343,18 +345,21 @@ ;; Parsing -(= (type 0) java.lang.Integer) - (defn maybe-parse-int ([s] (maybe-parse-int s 0)) - ([s default] - (if (= (type s) java.lang.Integer) - s - (try (Integer/parseInt s) - (catch NumberFormatException _ default))))) + ([s a] + (if (number? s) + (int s) + (try (Integer/parseInt s) + (catch NumberFormatException _ a))))) -(defn maybe-parse-long [s f] - (if s (Long/parseLong s) f)) +(defn maybe-parse-long + ([s] (maybe-parse-long s 0)) + ([s a] + (if (number? s) + (long s) + (try (Long/parseLong s) + (catch NumberFormatException _ a))))) (defn parse-yyyy-mm-dd-date [s] (try (.parse yyyy-mm-dd-formatter s) @@ -466,3 +471,11 @@ (swap! cached-results assoc arguments { :result result :time (System/currentTimeMillis)}) result))))) + +;; Taken from Programming Clojure by Stuart Halloway + +(defn index-filter [pred coll] + (for [[idx elt] (indexed coll) :when (pred elt)] idx)) + +(defn index-of [pred coll] + (first (index-filter pred coll))) |