From 90537b1498485f7571cedd47ed5700cd49f82afe Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Thu, 11 Nov 2010 13:34:49 -0500 Subject: Removed unused db functions --- src/site.clj | 72 ------------------------------------------------------------ 1 file changed, 72 deletions(-) diff --git a/src/site.clj b/src/site.clj index 3029f6a..6f97c81 100644 --- a/src/site.clj +++ b/src/site.clj @@ -110,34 +110,6 @@ (new-messages room ts)) "favs" (new-favs nick ts)}) -(defn count-messages-by-nick [nick image-only] - (let [query (str "SELECT COUNT(*) - FROM messages m, users u, rooms r - WHERE m.user_id = u.user_id AND u.nick = ? - AND r.room_id = m.room_id AND r.admin_only = false " - (if image-only "AND m.is_image = true " ""))] - (do-count [query nick]))) - -(defn fetch-messages-by-nick - ([nick image-only] (fetch-messages-by-nick nick image-only 0)) - ([nick image-only offset] - (let [query (str "SELECT m.content, m.created_on, m.message_id, u.nick, u.avatar, r.key - FROM messages m, users u, rooms r - WHERE m.user_id = u.user_id AND u.nick = ? - AND r.room_id = m.room_id AND r.admin_only = false " - (if image-only "AND m.is_image = true " "") - "ORDER BY created_on DESC - LIMIT ? OFFSET ?")] - (do-select [query nick *dumps-per-page* offset])))) - -(defn fetch-message-by-id [m-id] - (let [query "SELECT m.message_id, m.content, m.created_on, m.user_id, - u.nick, u.avatar, r.key, r.admin_only - FROM messages m, users u, rooms r - WHERE m.user_id = u.user_id - AND r.room_id = m.room_id - AND m.message_id = ?"] - (first (do-select [query (maybe-parse-int m-id -1)])))) (defn fetch-public-message-by-id [m-id] (let [msg (tags/fetch-dump-by-id m-id)] @@ -401,22 +373,6 @@ ORDER BY cnt DESC (map (comp take-images :content) dumps)))))) - -(defn count-dumps-posted [nick] - (:count - (first - (do-select ["select count(*) from messages m, users u - where m.user_id = u.user_id and lower(u.nick) = ? - and m.is_image = true" (.toLowerCase nick)])))) - -(defn count-dumps-user-faved [nick] - (:count - (first - (do-select ["select count(distinct(m.message_id)) from users u, tags t, messages m - where lower(u.nick) = ? and u.user_id = t.user_id - and t.tag = 'favorite' - and t.message_id = m.message_id and m.is_image = true" - (.toLowerCase nick)])))) (defn profile ([session profile-nick] (profile session profile-nick "profile")) @@ -602,34 +558,6 @@ order by count desc limit ? offset ?") "score_ent" (score-to-entity score) "score" score))) -(defn recent-posts-query [user-id] - (format " -SELECT u.user_id, u.nick, u.avatar, - m.content, m.message_id%s -FROM users u -LEFT JOIN messages m on m.message_id = - (SELECT message_id FROM messages - WHERE user_id = u.user_id - AND is_image - AND room_id IN (SELECT room_id from rooms where admin_only = false) - ORDER BY created_on desc LIMIT 1) -WHERE u.user_id = ANY(?)" - (if user-id - (format - ", - EXISTS (SELECT 1 FROM tags - WHERE tag = 'favorite' AND user_id = %s AND message_id = m.message_id) AS favorited" - user-id) - ", false AS favorited"))) - -(defn lookup-recent-posts [user-tag-id user-ids] - (do-select [(recent-posts-query user-tag-id) - (sql-array "int" user-ids)])) - -(defn lookup-recent-posts-tagless [user-tag-id user-ids] - (do-select [(recent-posts-query nil) - (sql-array "int" user-ids)])) - (def directory-cache-ttl (minutes 10)) (def memoized-lookup-recent-posts-tagless (ttl-memoize lookup-recent-posts-tagless directory-cache-ttl)) -- cgit v1.2.3-70-g09d2 From 7cd7b47e2e24c800e210d38e4f888bcb93cd438b Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Sun, 14 Nov 2010 00:22:56 -0800 Subject: cleaned up utility functions, added index-of func --- src/utils.clj | 43 ++++++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/src/utils.clj b/src/utils.clj index 08b06ab..8ed801e 100755 --- a/src/utils.clj +++ b/src/utils.clj @@ -18,6 +18,7 @@ clojure.contrib.sql clojure.contrib.def clojure.contrib.duck-streams + clojure.contrib.seq-utils clojure.contrib.str-utils compojure config)) @@ -82,11 +83,12 @@ (declare stringify-and-escape) (defn escape-html-deep [o] - (if (map? o) - (stringify-and-escape o) - (cond (seq? o) (map escape-html-deep o) - (or (true? o) (false? o)) o - :else (escape-html o)))) + (cond (map? o) (stringify-and-escape o) + (vector? o) (map escape-html-deep o) + (seq? o) (map escape-html-deep o) + (true? o) o + (false? o) o + :else (escape-html o))) (defn stringify-and-escape [m] (zipmap (map str* (keys m)) (map escape-html-deep (vals m)))) @@ -343,18 +345,21 @@ ;; Parsing -(= (type 0) java.lang.Integer) - (defn maybe-parse-int ([s] (maybe-parse-int s 0)) - ([s default] - (if (= (type s) java.lang.Integer) - s - (try (Integer/parseInt s) - (catch NumberFormatException _ default))))) - -(defn maybe-parse-long [s f] - (if s (Long/parseLong s) f)) + ([s a] + (if (number? s) + (int s) + (try (Integer/parseInt s) + (catch NumberFormatException _ a))))) + +(defn maybe-parse-long + ([s] (maybe-parse-long s 0)) + ([s a] + (if (number? s) + (long s) + (try (Long/parseLong s) + (catch NumberFormatException _ a))))) (defn parse-yyyy-mm-dd-date [s] (try (.parse yyyy-mm-dd-formatter s) @@ -463,3 +468,11 @@ (swap! cached-results assoc arguments { :result result :time (System/currentTimeMillis)}) result))))) + +;; Taken from Programming Clojure by Stuart Halloway + +(defn index-filter [pred coll] + (for [[idx elt] (indexed coll) :when (pred elt)] idx)) + +(defn index-of [pred coll] + (first (index-filter pred coll))) \ No newline at end of file -- cgit v1.2.3-70-g09d2 From e3a5d9b44762e32359a9652a6607fa7e56c874d7 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Sun, 14 Nov 2010 12:55:28 -0800 Subject: Updated config w/ redis info --- src/config.clj | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/config.clj b/src/config.clj index 3230c72..fdb33ed 100644 --- a/src/config.clj +++ b/src/config.clj @@ -11,10 +11,20 @@ "http://localhost:8080")) (def *cookie-domain* - (if (= *server-user* "timb") - "" - ".dump.fm")) + (if (= *server-user* "dumpfmprod") + ".dump.fm" + "")) + +(def redis-server + (if (= *server-user* "dumpfmprod") + {:host "192.168.156.111" :port 6379 :db 0 } + {:host "127.0.0.1" :port 6379 :db 0 })) (def *root-directory* (System/getProperty "user.dir")) (def *image-directory* "images") -(def *avatar-directory* "avatars") \ No newline at end of file +(def *avatar-directory* "avatars") + + +;; Numerical constants + +(def num-popular-dumps 40) -- cgit v1.2.3-70-g09d2 From 6b8215267bc5af44dfab4097992a31b35513b102 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Sun, 14 Nov 2010 12:55:55 -0800 Subject: Add redisload.clj for caching popular pages --- src/redisload.clj | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 src/redisload.clj diff --git a/src/redisload.clj b/src/redisload.clj new file mode 100644 index 0000000..fdc2664 --- /dev/null +++ b/src/redisload.clj @@ -0,0 +1,72 @@ +(ns redisload + (:use clojure.contrib.sql + config + datalayer + utils) + (:require redis)) + +(defn redis-days [n] + (* 24 60 60)) + +(def tag-query " +SELECT + u.nick as author, + u.user_id as author_id, + m.message_id as message_id, + m.content as message_content, + m.is_image as is_image, + m.created_on as message_ts, + r.key as room, + t.created_on as tagged_on, + u2.nick as tagger, + u2.user_id as tagger_id +FROM users u, messages m, rooms r, tags t, users u2 +WHERE + u.user_id = m.user_id AND + m.message_id = t.message_id AND + r.room_id = m.room_id AND + u2.user_id = t.user_id +") + +(def tag-counter (ref 0)) + +(defn stream-tags [fs] + (with-connection *db* + (with-query-results rs [tag-query] + (doseq [r rs] + (dosync + (ref-set tag-counter (inc @tag-counter))) + (doseq [f fs] + (f r)))))) + + +(def popular-map (ref {})) + +(defn update-popular [r] + (if (:is_image r) + (dosync + (let [user-map (or (get @popular-map (:author r)) {}) + msg-cnt (or (get user-map (:message_id r)) 0)] + (alter popular-map assoc (:author r) + (assoc user-map (:message_id r) (inc msg-cnt))))))) + +(defn transmit-popular [] + (doseq [[nick msgs] @popular-map] + (let [sorted-msgs (sort #(>= (second %1) (second %2)) msgs) + userkey (str "popular:" nick)] + (redis/atomically + (redis/del key) + (doseq [[msg-id score] (take (* num-popular-dumps 2) + sorted-msgs)] + (redis/zadd userkey score msg-id)) + ;; (redis/expire userkey (redis-days 2))))) -- need to schedule auto loader + (println "cached popular data for" (count @popular-map) "users")) + +(println "streaming tags") +(stream-tags [update-popular]) +(println (format "processed %s tags" @tag-counter)) + + +(redis/with-server redis-server + (transmit-popular)) + -- cgit v1.2.3-70-g09d2 From 1179b2a2aefb6a4c15db9634cc0c9c910fea8efa Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Sun, 14 Nov 2010 13:16:12 -0800 Subject: Added redis back for popular posts, fixed CTRL-M chars in src/site.clj, tweaked single_message.st --- src/datalayer.clj | 76 ++++++++++++++++++ src/site.clj | 186 +++++++++++---------------------------------- src/tags.clj | 60 +++++++-------- template/head.st | 4 +- template/popular_dump.st | 2 +- template/single_message.st | 56 ++++++++------ 6 files changed, 186 insertions(+), 198 deletions(-) create mode 100644 src/datalayer.clj diff --git a/src/datalayer.clj b/src/datalayer.clj new file mode 100644 index 0000000..7086b12 --- /dev/null +++ b/src/datalayer.clj @@ -0,0 +1,76 @@ +(ns datalayer + (:require redis + tags) + (:use config + utils)) + + + +;;;; Message lookup + +(defn recent-posts-query [user-id] + (format " +SELECT u.user_id, u.nick, u.avatar, + m.content, m.message_id%s +FROM users u +LEFT JOIN messages m on m.message_id = + (SELECT message_id FROM messages + WHERE user_id = u.user_id + AND is_image + AND room_id IN (SELECT room_id from rooms where admin_only = false) + ORDER BY created_on desc LIMIT 1) +WHERE u.user_id = ANY(?)" + (if user-id + (format + ", + EXISTS (SELECT 1 FROM tags + WHERE tag = 'favorite' AND user_id = %s AND message_id = m.message_id) AS favorited" + user-id) + ", false AS favorited"))) + +(defn lookup-recent-posts [user-tag-id user-ids] + (do-select [(recent-posts-query user-tag-id) + (sql-array "int" user-ids)])) + +(defn lookup-recent-posts-tagless [user-tag-id user-ids] + (do-select [(recent-posts-query nil) + (sql-array "int" user-ids)])) + +(defn fetch-message-by-id [m-id] + (let [query "SELECT m.message_id, m.content, m.created_on, m.user_id, + u.nick, u.avatar, r.key, r.admin_only + FROM messages m, users u, rooms r + WHERE m.user_id = u.user_id + AND r.room_id = m.room_id + AND m.message_id = ?"] + (first (do-select [query (maybe-parse-int m-id -1)])))) + + + +;;;; Popular Posts + +(def popular-dumps-qry " +select u.nick, u.avatar, r.key, m.message_id, m.content, m.created_on, count(*) as count, + array_agg(u2.nick) as user_nicks +from users u, messages m, rooms r, tags t, users u2 +where lower(u.nick) = lower(?) +and u.user_id = m.user_id and m.message_id = t.message_id +and m.room_id = r.room_id and m.is_image = true and r.admin_only = false +and t.tag = 'favorite' +and t.user_id = u2.user_id +group by u.nick, u.avatar, r.key, m.message_id, m.content, m.created_on +order by count desc limit ? offset ?") + +(defn fetch-popular-dumps [nick viewer-nick] + (for [d (do-select [popular-dumps-qry nick 40 0])] + (let [favers (vec (.getArray (:user_nicks d)))] + (assoc d + :favers favers + :favorited (some #(= % viewer-nick) favers))))) + +(defn fetch-popular-dumps-redis [nick viewer-nick] + (let [rkey (str "popular:" nick) + msg-ids (redis/with-server redis-server + (redis/zrevrange rkey 0 (dec num-popular-dumps))) + msg-ids (map maybe-parse-int msg-ids)] + (tags/fetch-dumps-by-ids msg-ids viewer-nick))) \ No newline at end of file diff --git a/src/site.clj b/src/site.clj index 161f0d5..1abb876 100644 --- a/src/site.clj +++ b/src/site.clj @@ -18,6 +18,7 @@ config admin compojure + datalayer email fame utils @@ -110,38 +111,10 @@ (new-messages room ts)) "favs" (new-favs nick ts)}) -(defn count-messages-by-nick [nick image-only] - (let [query (str "SELECT COUNT(*) - FROM messages m, users u, rooms r - WHERE m.user_id = u.user_id AND u.nick = ? - AND r.room_id = m.room_id AND r.admin_only = false " - (if image-only "AND m.is_image = true " ""))] - (do-count [query nick]))) - -(defn fetch-messages-by-nick - ([nick image-only] (fetch-messages-by-nick nick image-only 0)) - ([nick image-only offset] - (let [query (str "SELECT m.content, m.created_on, m.message_id, u.nick, u.avatar, r.key - FROM messages m, users u, rooms r - WHERE m.user_id = u.user_id AND u.nick = ? - AND r.room_id = m.room_id AND r.admin_only = false " - (if image-only "AND m.is_image = true " "") - "ORDER BY created_on DESC - LIMIT ? OFFSET ?")] - (do-select [query nick *dumps-per-page* offset])))) - -(defn fetch-message-by-id [m-id] - (let [query "SELECT m.message_id, m.content, m.created_on, m.user_id, - u.nick, u.avatar, r.key, r.admin_only - FROM messages m, users u, rooms r - WHERE m.user_id = u.user_id - AND r.room_id = m.room_id - AND m.message_id = ?"] - (first (do-select [query (maybe-parse-int m-id -1)])))) - -(defn fetch-public-message-by-id [m-id] - (let [msg (tags/fetch-dump-by-id m-id)] - (if (and msg (not (:admin_only msg))) + +(defn fetch-public-message-by-id [m-id viewer-nick] + (if-let [msg (tags/fetch-dump-by-id m-id viewer-nick)] + (if-not (:admin_only msg) msg))) ;; User-id/nick cache @@ -246,9 +219,10 @@ (defn log-login [user_id ip] ;; i'm using do-cmds here because update-values can't deal with stuff like 'last_login = now()' - (let [query (format "UPDATE users SET last_ip = '%s'::cidr, last_login = now() WHERE user_id = %s" (str ip) (str user_id))] - (do-cmds query)) -) + (try + (let [query (format "UPDATE users SET last_ip = '%s'::cidr, last_login = now() WHERE user_id = %s" (str ip) (str user_id))] + (do-cmds query)) + (catch Exception e nil))) (defn login [session params cookies request] (let [nick (or (params :nick) "") @@ -264,9 +238,8 @@ (log-login (db-user :user_id) ip) [(session-assoc-from-db db-user) login-cookie - (resp-success "OK")] - ) - (resp-error "BAD_LOGIN")))) + (resp-success "OK")]) + (resp-error "BAD_LOGIN")))) (defn logout [session] [(session-dissoc :nick :user_id :is_admin :avatar) @@ -401,22 +374,6 @@ ORDER BY cnt DESC (map (comp take-images :content) dumps)))))) - -(defn count-dumps-posted [nick] - (:count - (first - (do-select ["select count(*) from messages m, users u - where m.user_id = u.user_id and lower(u.nick) = ? - and m.is_image = true" (.toLowerCase nick)])))) - -(defn count-dumps-user-faved [nick] - (:count - (first - (do-select ["select count(distinct(m.message_id)) from users u, tags t, messages m - where lower(u.nick) = ? and u.user_id = t.user_id - and t.tag = 'favorite' - and t.message_id = m.message_id and m.is_image = true" - (.toLowerCase nick)])))) (defn profile ([session profile-nick] (profile session profile-nick "profile")) @@ -549,42 +506,15 @@ ORDER BY cnt DESC ;; Who faved me -(def popular-dumps-qry " -select u.nick, u.avatar, r.key, m.message_id, m.content, m.created_on, count(*) as count, - array_agg(u2.nick) as user_nicks, - array_agg(u2.avatar) as user_avs, - array_agg(t.created_on) as favtime, - (select exists (select 1 from tags - where tag = 'favorite' and user_id = ? and message_id = m.message_id)) as favorited -from users u, messages m, rooms r, tags t, users u2 -where lower(u.nick) = lower(?) -and u.user_id = m.user_id and m.message_id = t.message_id -and m.room_id = r.room_id and m.is_image = true and r.admin_only = false -and t.tag = 'favorite' and t.user_id != u.user_id -and t.user_id = u2.user_id -group by u.nick, u.avatar, r.key, m.message_id, m.content, m.created_on -order by count desc limit ? offset ?") - -(def num-popular-dumps 40) - -(defn get-popular-dumps [nick user-id] - (for [d (do-select [popular-dumps-qry user-id nick 40 0])] - (let [fav-nicks (.getArray (:user_nicks d))] - (assoc d - :favers (sort-by :t (comp #(* -1 %) compare) - (map (fn [n a t] (if (non-empty-string? a) - {:nick n :avatar a :t t} - {:nick n :t t})) - fav-nicks - (.getArray (:user_avs d)) - (.getArray (:favtime d)))) - :user_nicks nil :user_avs nil :favtime nil)))) +(def use-popular-redis false) (defn popular [session profile-nick] (if-let [user-info (fetch-nick profile-nick)] (let [st (fetch-template "popular" session) profile-nick (:nick user-info) - raw-dumps (get-popular-dumps profile-nick (or (:user_id session) -1)) + raw-dumps (if use-popular-redis + (fetch-popular-dumps-redis profile-nick (:nick session)) + (fetch-popular-dumps profile-nick (:nick session))) dumps (map process-message-for-output raw-dumps)] (.setAttribute st "nick" profile-nick) (.setAttribute st "mini_profile" (build-mini-profile user-info)) @@ -602,33 +532,6 @@ order by count desc limit ? offset ?") "score_ent" (score-to-entity score) "score" score))) -(defn recent-posts-query [user-id] - (format " -SELECT u.user_id, u.nick, u.avatar, - m.content, m.message_id%s -FROM users u -LEFT JOIN messages m on m.message_id = - (SELECT message_id FROM messages - WHERE user_id = u.user_id - AND is_image - AND room_id IN (SELECT room_id from rooms where admin_only = false) - ORDER BY created_on desc LIMIT 1) -WHERE u.user_id = ANY(?)" - (if user-id - (format - ", - EXISTS (SELECT 1 FROM tags - WHERE tag = 'favorite' AND user_id = %s AND message_id = m.message_id) AS favorited" - user-id) - ", false AS favorited"))) - -(defn lookup-recent-posts [user-tag-id user-ids] - (do-select [(recent-posts-query user-tag-id) - (sql-array "int" user-ids)])) - -(defn lookup-recent-posts-tagless [user-tag-id user-ids] - (do-select [(recent-posts-query nil) - (sql-array "int" user-ids)])) (def directory-cache-ttl (minutes 10)) (def memoized-lookup-recent-posts-tagless @@ -662,19 +565,15 @@ WHERE u.user_id = ANY(?)" ;; Single posts (defn single-message [session nick-from-url id-from-url] - (if-let [user-info (fetch-nick nick-from-url)] - (if-let [message (fetch-public-message-by-id id-from-url)] - ; error if nick in url doesn't match the nick who posted the message from the id in url - ; this prevents people from scraping all the content by incrementing the id in the url - (if (= (user-info :user_id) (message :user_id)) - (let [st (fetch-template "single_message" session) - message (tags/add-favorited-flag message session) - message (tags/remove-tags-for-output message)] - (.setAttribute st "dump" (process-message-for-output message)) - (.toString st)) - (resp-error "NO_MESSAGE")) + (if-let [message (fetch-public-message-by-id id-from-url (:nick session))] + ; error if nick in url doesn't match the nick who posted the message from the id in url + ; this prevents people from scraping all the content by incrementing the id in the url + (if (= nick-from-url (:nick message)) + (let [st (fetch-template "single_message" session)] + (.setAttribute st "dump" (process-message-for-output message)) + (.toString st)) (resp-error "NO_MESSAGE")) - (resp-error "NO_USER"))) + (resp-error "NO_MESSAGE"))) ;; Chat @@ -844,18 +743,15 @@ WHERE u.user_id = ANY(?)" (.toString st))) (defn validated-log [session room-key offset params] - (if-vip - (let [room-key (if (= (lower-case room-key) "www") "dumpfm" room-key)] - (if (validate-room-access room-key session) - (log session (lookup-room room-key) offset params) - (resp-error "UNKNOWN_ROOM"))) - (redirect-to "http://dump.fm"))) + (let [room-key (if (= (lower-case room-key) "www") "dumpfm" room-key)] + (if (validate-room-access room-key session) + (log session (lookup-room room-key) offset params) + (resp-error "UNKNOWN_ROOM")))) ;; Hiscore test... redis test... (defn redis-ids-test [period] - (let [reddis-server {:host "127.0.0.1" :port 6379 :db 0} - ids (redis/with-server reddis-server + (let [ids (redis/with-server redis-server (redis/zrevrange (str "hiscore:" period) 0 -1)) ids (map maybe-parse-int ids)] ids)) @@ -1070,15 +966,14 @@ WHERE u.user_id = ANY(?)" ;; cons: has to use a + + @@ -10,8 +12,6 @@ $if(!user_nick)$ - - $endif$ $if(isadmin)$ diff --git a/template/popular_dump.st b/template/popular_dump.st index 4709725..88e61c4 100644 --- a/template/popular_dump.st +++ b/template/popular_dump.st @@ -9,7 +9,7 @@ $dump.created_on$ - in $dump.key$$dump.content$
-$dump.count$ $dump.favers: { f | $f.nick$   }$ +$dump.count$ $dump.favers: { nick | $nick$   }$
$share_buttons()$ diff --git a/template/single_message.st b/template/single_message.st index e017b6f..1428f2b 100644 --- a/template/single_message.st +++ b/template/single_message.st @@ -1,37 +1,47 @@ - dump.fm $head()$ - - $banner()$ -


-
-
-$if(dump.avatar)$ -
-
- -
- $dump.nick$ -
-
-$endif$ - -
$dump.content$
-
-$share_buttons()$ +



+
+ + - + + +
$dump.content$
+
+
+ $dump.count$ $dump.favers: { nick | $nick$   }$ +
+$share_buttons()$ +
+ + +
+ $footer()$ +
+
+ -- cgit v1.2.3-70-g09d2 From 789ed39751c17fa1aa73f3119a80403e34f0d984 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Sun, 14 Nov 2010 13:19:49 -0800 Subject: Disable scores/hall updating --- src/fame.clj | 4 ++-- src/site.clj | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/fame.clj b/src/fame.clj index 0732992..1baf6e0 100644 --- a/src/fame.clj +++ b/src/fame.clj @@ -2,8 +2,8 @@ (:use scheduled-agent utils)) -; should probably cache this to disk somewhere -(def hall-of-fame-update-frequency (* 5 60 60)) +; sostler: don't refresh until redis cache can be added +(def hall-of-fame-update-frequency (* 5 60 60 999999)) (def hall-of-fame-query " select m.created_on, m.message_id, m.content, u.nick, u.avatar, r.key, count(*) diff --git a/src/site.clj b/src/site.clj index 1abb876..4c3560e 100644 --- a/src/site.clj +++ b/src/site.clj @@ -328,7 +328,8 @@ ORDER BY cnt DESC {:list res :map (zipmap (map :nick res) (map :cnt res))})) -(def *scores-refresh-period-sec* (* 29 60)) +;; sostler: stop score refresh until redis cache can be added +(def *scores-refresh-period-sec* (* 29 60 9999999)) (def *user-scores* (scheduled-agent build-score-list -- cgit v1.2.3-70-g09d2 From 2041ed45c75ded3b0f7b6c5e6536dc262406b0d1 Mon Sep 17 00:00:00 2001 From: dumpfmprod Date: Sun, 14 Nov 2010 16:22:43 -0500 Subject: sostler prod commit --- static/css/dump.css | 6 +- static/css/front.css | 162 +++++++++++++++++++++++++++++++++++++++++++++ static/img/sassoonlogo.gif | Bin 0 -> 31936 bytes template/3frames.st | 15 +++++ template/banner.st | 9 +-- template/fame2.st | 4 +- template/head.st | 2 +- template/profile.st | 4 +- 8 files changed, 191 insertions(+), 11 deletions(-) create mode 100644 static/css/front.css create mode 100644 static/img/sassoonlogo.gif create mode 100644 template/3frames.st diff --git a/static/css/dump.css b/static/css/dump.css index df3e55e..ce07b45 100755 --- a/static/css/dump.css +++ b/static/css/dump.css @@ -151,10 +151,11 @@ height:22; text-indent:14px; line-height:1.9; left: 125px; - margin-left: 0.1%; + margin-left: 5%; margin-right: 8%; letter-spacing:.2px; } +#sassoon{margin-left:-15;margin-top:2;} #bar7dis{ top:19px; position:absolute; @@ -912,7 +913,6 @@ border:1px solid #000;border-right:0px; max-height:400px; height: expression(this.width > 400 ? 400: true); max-width:400px; - margin:-2; opacity:1; } @@ -1346,7 +1346,7 @@ margin-left:-5px; font-size:12px; } - +.img-wrapper img{margin-right:-4px;} #logavatar img{ max-width:20px; diff --git a/static/css/front.css b/static/css/front.css new file mode 100644 index 0000000..1bf973e --- /dev/null +++ b/static/css/front.css @@ -0,0 +1,162 @@ +#login-container { + float: right; + /*width:780px; + margin:0 auto;*/ +right:3; +color:#fff; + position: relative; +top:-2; +filter:alpha(opacity=90); + -moz-opacity:0.9; + -khtml-opacity: 0.9; + opacity: 0.9z-index:2; + +} + +#login-container a:link, a:visited { + color:#000; + text-decoration:none; +} +#login-container .topnav { + padding:0px 0px 12px; + font-size:11px;color:000; + line-height:23px; + text-shadow: #000 0px 0px 0px; + text-align:right; + +} +#login-container .topnav a.signin { + text-shadow: #000 0px 0px 0px; + padding:5px 5px 5px 5px; + text-decoration:none; + background-color:#fff; + + + + *padding:4px 12px 6px; filter:alpha(opacity=80); + + +} +#login-container .topnav a.signin:hover { + + *padding:4px 12px 16px; background-color:#ccc; + +} +#login-container .topnav a.signin, #login-container .topnav a.signin:hover { + *background-position:0 3px!important;color:#000; +} + +a.signin { + position:relative; +cursor:pointer; + margin-left:3px; +} +a.signin span { + background-image:url("http://dump.fm/static/form_login/images/toggle_down_light.png"); + background-repeat:no-repeat; + background-position:100% 50%; + padding:4px 16px 6px 0;cursor:pointer; +} +#login-container .topnav a.menu-open { + background:#ddeef6!important; + color:#666!important; text-shadow: #000 0px 0px 0px; + outline:none;box-shadow: 3px 2px 2px #ccc;cursor:pointer; +-webkit-box-shadow: 3px 2px 2px #ccc; +-moz-box-shadow: 3px 2px 2px #ccc; +filter: progid:DXImageTransform.Microsoft.dropShadow(color=#ccc, offX=3, offY=4, positive=true); + +} + +a.signin.menu-open span {cursor:pointer; + background-image:url("http://dump.fm/static/form_login/images/toggle_up_dark.png"); + color:#789; +} + +#signin_menu { + -moz-border-radius-topleft:5px; + -moz-border-radius-bottomleft:5px; + -moz-border-radius-bottomright:5px; + -webkit-border-top-left-radius:5px; + -webkit-border-bottom-left-radius:5px; + -webkit-border-bottom-right-radius:5px; + display:none; + background-color:#ddeef6; + position:absolute; + width:210px; + z-index:100; + border:1px transparent; + text-align:left; + padding:12px; + top: 16.5px; + right: 0px; + margin-top:5px; + margin-right: 0px; + *margin-right: -1px; + color:#789; + font-size:11px; box-shadow: 3px 2px 2px #ccc; +-webkit-box-shadow: 3px 2px 2px #ccc; +z-index:5000; +-moz-box-shadow: 3px 2px 2px #ccc; +filter: progid:DXImageTransform.Microsoft.dropShadow(color=#ccc, offX=3, offY=4, positive=true); +} + +#signin_menu input[type=text], #signin_menu input[type=password] { + display:block; + font-size:25px; + margin:0 0 5px; + padding:5px; + width:203px; +} +#signin_menu p { + margin:0; +} +#signin_menu a { + color:#6AC; +} +#signin_menu label { + font-weight:normal; +} + +#signin_menu p a { + color:#27B!important; +} +#signin-submit { + display:inline-block; + width:90px; + height:35px; + font-size:20px; + background-image:url('http://dump.fm/static/img/btngrad1.png'); + background-color:#4691e5; + font-weight:bold; + word-spacing:7; + margin-top:-5px; +cursor:pointer; + text-align:center; + z-index:100; + font-size:16px; + color:#fff; + border-radius: 5px; + -webkit-border-radius: 5px; + -moz-border-radius: 5px; + text-shadow:1px 1px 3px #000; +z-index:20; +} + +#furie3 { + position: absolute; + left:294px; + margin-top:250px; + z-index: 10; + +} +#loginSubmit::-moz-focus-inner { +padding:0; +border:0; +} +#loginSubmit:hover, #loginSubmit:focus { + background-position:0 -5px; + cursor:pointer; +} +.no-cursor { cursor: none; } +.invisible { display: none !important; } +#cursor-big { position: absolute; z-index: 1000; } \ No newline at end of file diff --git a/static/img/sassoonlogo.gif b/static/img/sassoonlogo.gif new file mode 100644 index 0000000..29cf6ed Binary files /dev/null and b/static/img/sassoonlogo.gif differ diff --git a/template/3frames.st b/template/3frames.st new file mode 100644 index 0000000..3fd00a8 --- /dev/null +++ b/template/3frames.st @@ -0,0 +1,15 @@ + + dump.fm - Deal With It GIF Maker + $head()$ + + + + $banner()$ + +
+
+ +
+ $footer()$ +
+ \ No newline at end of file diff --git a/template/banner.st b/template/banner.st index 615256f..6108c9a 100644 --- a/template/banner.st +++ b/template/banner.st @@ -3,8 +3,8 @@
- - + +
@@ -27,8 +27,8 @@ Hall of Fame - - Favs + Image Search @@ -39,6 +39,7 @@ + diff --git a/template/fame2.st b/template/fame2.st index 89739cd..debc13e 100644 --- a/template/fame2.st +++ b/template/fame2.st @@ -15,14 +15,14 @@
-

Hall Of Fame


updated hourly


+

Hall Of Fame



VIEW BY -
+
$if(dumps)$ diff --git a/template/head.st b/template/head.st index ad23a44..2cfd6cb 100644 --- a/template/head.st +++ b/template/head.st @@ -8,7 +8,7 @@ $if(!user_nick)$ - + diff --git a/template/profile.st b/template/profile.st index 8d8a019..d948e4f 100644 --- a/template/profile.st +++ b/template/profile.st @@ -74,8 +74,10 @@

$if(is_home)$ your $else$ $nick$'s$endif$ posts ➡

$if(is_home)$ your $else$ $nick$'s $endif$ favs ➡

-
+
$if(is_home)$ your $else$ $nick$'s$endif$ most recent dumps
 -- cgit v1.2.3-70-g09d2 From 9b5fb453d7fbe79918fe012badbf6a515092cb48 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Sun, 14 Nov 2010 13:26:44 -0800 Subject: Fixed redisload typo --- src/redisload.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redisload.clj b/src/redisload.clj index fdc2664..9e33039 100644 --- a/src/redisload.clj +++ b/src/redisload.clj @@ -58,7 +58,7 @@ WHERE (redis/del key) (doseq [[msg-id score] (take (* num-popular-dumps 2) sorted-msgs)] - (redis/zadd userkey score msg-id)) + (redis/zadd userkey score msg-id))))) ;; (redis/expire userkey (redis-days 2))))) -- need to schedule auto loader (println "cached popular data for" (count @popular-map) "users")) -- cgit v1.2.3-70-g09d2 From c37bcf6d26abf3fdadd41deda24b020b71627630 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Mon, 15 Nov 2010 01:54:43 -0800 Subject: Moved constants to config.clj --- src/config.clj | 3 +++ src/rooms.clj | 2 -- src/site.clj | 2 +- src/utils.clj | 3 --- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/config.clj b/src/config.clj index fdb33ed..7a4a6a2 100644 --- a/src/config.clj +++ b/src/config.clj @@ -28,3 +28,6 @@ ;; Numerical constants (def num-popular-dumps 40) +(def *dumps-per-page* 20) +(def *vip-dumps-per-page* 200) +(def message-count-limit 200) diff --git a/src/rooms.clj b/src/rooms.clj index e919557..7745630 100644 --- a/src/rooms.clj +++ b/src/rooms.clj @@ -101,8 +101,6 @@ (defn build-msg [nick content msg-id] (struct message-struct nick content (new Date) msg-id)) -(def message-count-limit 200) - (defn add-message [msg room] (insert-and-truncate! (room :messages) msg message-count-limit)) diff --git a/src/site.clj b/src/site.clj index 4c3560e..5d7f9a3 100644 --- a/src/site.clj +++ b/src/site.clj @@ -677,7 +677,7 @@ ORDER BY cnt DESC mute (resp-error (format-mute mute)) :else (let [content (validated-content content session) - msg-id (msg-db user-id (room :room_id) content)] + msg-id (msg-db user-id (room :room_id) content)] (dosync (if (not (contains? (ensure (room :users)) nick)) (login-user (user-struct-from-session session) room)) diff --git a/src/utils.clj b/src/utils.clj index 9d7fd3a..84454cd 100755 --- a/src/utils.clj +++ b/src/utils.clj @@ -35,9 +35,6 @@ (.setPassword db-pass) (.setMaxConnections 10))})) -;; moved this to here which doesn't seem right... maybe a 'settings.clj' or something? -(def *dumps-per-page* 20) -(def *vip-dumps-per-page* 200) ;; Message parsing -- cgit v1.2.3-70-g09d2 From e9e057b6231a7544e3f7fb2007abe23c7aab224a Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Wed, 17 Nov 2010 01:09:39 -0500 Subject: Add check for empty fetch-popular-dumps-redis --- src/datalayer.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/datalayer.clj b/src/datalayer.clj index 7086b12..38a597e 100644 --- a/src/datalayer.clj +++ b/src/datalayer.clj @@ -73,4 +73,5 @@ order by count desc limit ? offset ?") msg-ids (redis/with-server redis-server (redis/zrevrange rkey 0 (dec num-popular-dumps))) msg-ids (map maybe-parse-int msg-ids)] - (tags/fetch-dumps-by-ids msg-ids viewer-nick))) \ No newline at end of file + (if-not (empty? msg-ids) + (tags/fetch-dumps-by-ids msg-ids viewer-nick)))) -- cgit v1.2.3-70-g09d2 From 2e1afb150bb28c26f63392ea7dc37567c1950528 Mon Sep 17 00:00:00 2001 From: dumpfmprod Date: Wed, 17 Nov 2010 01:10:42 -0500 Subject: enable redis popular posts --- src/site.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/site.clj b/src/site.clj index 4c3560e..c492336 100644 --- a/src/site.clj +++ b/src/site.clj @@ -507,7 +507,7 @@ ORDER BY cnt DESC ;; Who faved me -(def use-popular-redis false) +(def use-popular-redis true) (defn popular [session profile-nick] (if-let [user-info (fetch-nick profile-nick)] -- cgit v1.2.3-70-g09d2 From b31b524d42cb2cb602b8e8c52e7c27a968770cc4 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Wed, 17 Nov 2010 01:12:29 -0500 Subject: use config.clj in rooms/tags --- src/config.clj | 2 +- src/rooms.clj | 1 + src/tags.clj | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/config.clj b/src/config.clj index 7a4a6a2..ce3a88c 100644 --- a/src/config.clj +++ b/src/config.clj @@ -25,7 +25,7 @@ (def *avatar-directory* "avatars") -;; Numerical constants +;; Settings (def num-popular-dumps 40) (def *dumps-per-page* 20) diff --git a/src/rooms.clj b/src/rooms.clj index 7745630..9855a59 100644 --- a/src/rooms.clj +++ b/src/rooms.clj @@ -2,6 +2,7 @@ (:import java.util.Date) (:use clojure.contrib.str-utils clojure.contrib.def + config utils user)) diff --git a/src/tags.clj b/src/tags.clj index a8c3341..37c93b0 100644 --- a/src/tags.clj +++ b/src/tags.clj @@ -6,6 +6,7 @@ clojure.contrib.fcase clojure.contrib.json.write clojure.contrib.str-utils + config compojure utils)) -- cgit v1.2.3-70-g09d2 From 52090faa3ca95bbcb1bcea638834f72cf3d93e87 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Fri, 19 Nov 2010 00:55:06 -0500 Subject: redisload.clj: hide admin only --- src/redisload.clj | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/redisload.clj b/src/redisload.clj index 9e33039..0e34e2c 100644 --- a/src/redisload.clj +++ b/src/redisload.clj @@ -17,6 +17,7 @@ SELECT m.is_image as is_image, m.created_on as message_ts, r.key as room, + r.admin_only as admin_only, t.created_on as tagged_on, u2.nick as tagger, u2.user_id as tagger_id @@ -43,7 +44,7 @@ WHERE (def popular-map (ref {})) (defn update-popular [r] - (if (:is_image r) + (if (and (:is_image r) (not (:admin_only r))) (dosync (let [user-map (or (get @popular-map (:author r)) {}) msg-cnt (or (get user-map (:message_id r)) 0)] @@ -59,7 +60,7 @@ WHERE (doseq [[msg-id score] (take (* num-popular-dumps 2) sorted-msgs)] (redis/zadd userkey score msg-id))))) - ;; (redis/expire userkey (redis-days 2))))) -- need to schedule auto loader + ;; (redis/expire userkey (redis-days 2))))) -- TODO: schedule auto loader (println "cached popular data for" (count @popular-map) "users")) (println "streaming tags") -- cgit v1.2.3-70-g09d2 From 8192ea8c0ac6d5579e519dd47a7921b93301df95 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Sat, 20 Nov 2010 16:26:49 -0500 Subject: Add show vip favorites for vips --- src/site.clj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/site.clj b/src/site.clj index 5d7f9a3..3f7b462 100644 --- a/src/site.clj +++ b/src/site.clj @@ -907,12 +907,14 @@ ORDER BY cnt DESC :nick (:nick user-info) :user-tag-id (:user_id session) :msg-id msg-id + :hide-vip (not (:is_admin session)) :date (if msg-id nil date) :limit (inc *dumps-per-page*)) back-dumps (if (or date msg-id) (tags/fetch-tagged-dumps :nick (:nick user-info) :msg-id msg-id + :hide-vip (not (:is_admin session)) :date (if msg-id nil date) :limit (inc *dumps-per-page*) :direction :forward)) -- cgit v1.2.3-70-g09d2 From 9e48126c34a821e44058ebc37f6e9d31ae1986eb Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Sun, 21 Nov 2010 17:36:53 -0500 Subject: remove useless todo --- src/rooms.clj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/rooms.clj b/src/rooms.clj index 9855a59..574532a 100644 --- a/src/rooms.clj +++ b/src/rooms.clj @@ -122,7 +122,6 @@ (build-room-map-from-db room-db)) room-db))) -; TODO: cache (defn get-or-create-room! [key] (:room_id (or (first (do-select ["SELECT room_id FROM rooms WHERE lower(key) = ?" -- cgit v1.2.3-70-g09d2 From 572d443a91dc244a1bdc1fb930060d71e7ebc243 Mon Sep 17 00:00:00 2001 From: dumpfmprod Date: Sun, 21 Nov 2010 21:43:06 -0500 Subject: sostler css commit --- static/css/dump.css | 6 +++--- template/profile.st | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/static/css/dump.css b/static/css/dump.css index ce07b45..d4b7000 100755 --- a/static/css/dump.css +++ b/static/css/dump.css @@ -140,7 +140,7 @@ left:-1px; box-shadow: 0 0 10px #d8dbde, 0px 0px 2px #000; -webkit-box-shadow: 0 0 10px #d8dbde, 0px 0px 2px #000; -moz-box-shadow: 0 0 10px #d8dbde, 0px 0px 2px #000; - +z-index: 1; } #bar7{ top:19px; @@ -1622,7 +1622,7 @@ z-index:5555; max-height:600px; height: expression(this.width > 600 ? 600: true); border:0px; - z-index:4; + z-index:0; } @@ -1642,7 +1642,7 @@ text-overflow: ellipsis-word; border:1px solid #f0e0d6; margin-left:20px; margin-bottom:20px; - z-index:4; + z-index:0; line-height:20px; text-align: left; diff --git a/template/profile.st b/template/profile.st index d948e4f..f66069e 100644 --- a/template/profile.st +++ b/template/profile.st @@ -74,10 +74,10 @@

$if(is_home)$ your $else$ $nick$'s$endif$ posts ➡

$if(is_home)$ your $else$ $nick$'s $endif$ favs ➡

- +
$if(is_home)$ your $else$ $nick$'s$endif$ most recent dumps
 -- cgit v1.2.3-70-g09d2 From 8c43debe825b0992322b2a6f2a77a6ce62d33af4 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Tue, 23 Nov 2010 01:23:39 -0500 Subject: Add jedis library --- bin/repl.sh | 2 +- lib/commons-pool-1.5.5.jar | Bin 0 -> 100193 bytes lib/jedis-1.4.0.jar | Bin 0 -> 98935 bytes src/jedis.clj | 15 +++++++++++++++ 4 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 lib/commons-pool-1.5.5.jar create mode 100644 lib/jedis-1.4.0.jar create mode 100644 src/jedis.clj diff --git a/bin/repl.sh b/bin/repl.sh index 1b5bca1..9edeaaa 100755 --- a/bin/repl.sh +++ b/bin/repl.sh @@ -1,3 +1,3 @@ #!/bin/sh -java -Xmx320m -server -cp .:lib/commons-io-1.4.jar:lib/commons-fileupload-1.2.1.jar:lib/commons-codec-1.3.jar:lib/jline-0.9.94.jar:lib/clojure.jar:lib/clojure-contrib.jar:lib/compojure-3.2v3.jar:lib/jetty-6.1.24.jar:lib/jetty-util-6.1.24.jar:lib/servlet-api-2.5-6.1.14.jar:lib/postgresql-8.4-701.jdbc4.jar:lib/stringtemplate-3.2.1.jar:lib/antlr-2.7.7.jar:lib/mail-1.4.4.jar:lib/jdom-1.1.jar:lib/rome-1.0.jar:lib/htmlcleaner-2.1.jar:lib/redis-clojure-1.0.4.jar:src/ jline.ConsoleRunner clojure.main -i $1 -r $@ \ No newline at end of file +java -Xmx320m -server -cp .:lib/commons-io-1.4.jar:lib/commons-fileupload-1.2.1.jar:lib/commons-codec-1.3.jar:lib/commons-pool-1.5.5.jar:lib/jline-0.9.94.jar:lib/clojure.jar:lib/clojure-contrib.jar:lib/compojure-3.2v3.jar:lib/jetty-6.1.24.jar:lib/jetty-util-6.1.24.jar:lib/servlet-api-2.5-6.1.14.jar:lib/postgresql-8.4-701.jdbc4.jar:lib/stringtemplate-3.2.1.jar:lib/antlr-2.7.7.jar:lib/mail-1.4.4.jar:lib/jdom-1.1.jar:lib/rome-1.0.jar:lib/htmlcleaner-2.1.jar:lib/redis-clojure-1.0.4.jar:lib/jedis-1.4.0.jar:src/ jline.ConsoleRunner clojure.main -i $1 -r $@ \ No newline at end of file diff --git a/lib/commons-pool-1.5.5.jar b/lib/commons-pool-1.5.5.jar new file mode 100644 index 0000000..7a96587 Binary files /dev/null and b/lib/commons-pool-1.5.5.jar differ diff --git a/lib/jedis-1.4.0.jar b/lib/jedis-1.4.0.jar new file mode 100644 index 0000000..a7d5757 Binary files /dev/null and b/lib/jedis-1.4.0.jar differ diff --git a/src/jedis.clj b/src/jedis.clj new file mode 100644 index 0000000..bc53eda --- /dev/null +++ b/src/jedis.clj @@ -0,0 +1,15 @@ +(ns jedis + (:import org.apache.commons.pool.impl.GenericObjectPool$Config + redis.clients.jedis.JedisPool) + (:use config)) + +(def pool (JedisPool. (:host redis-server) (:port redis-server))) +(.init pool) + + +(defn with-jedis [fn] + (let [r (.getResource pool)] + (try + (fn r) + (finally (.returnResource pool r))))) + -- cgit v1.2.3-70-g09d2 From 2b7162504314d4207df4c462d3cad12ad144b7c0 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Tue, 23 Nov 2010 01:23:57 -0500 Subject: Add directory/hall to redis loader --- src/redisload.clj | 45 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/src/redisload.clj b/src/redisload.clj index 0e34e2c..07c1275 100644 --- a/src/redisload.clj +++ b/src/redisload.clj @@ -1,5 +1,6 @@ (ns redisload (:use clojure.contrib.sql + clojure.contrib.str-utils config datalayer utils) @@ -26,7 +27,8 @@ WHERE u.user_id = m.user_id AND m.message_id = t.message_id AND r.room_id = m.room_id AND - u2.user_id = t.user_id + u2.user_id = t.user_id AND + r.admin_only = false ") (def tag-counter (ref 0)) @@ -35,21 +37,30 @@ WHERE (with-connection *db* (with-query-results rs [tag-query] (doseq [r rs] + (if (:admin_only r) + (println r)) (dosync (ref-set tag-counter (inc @tag-counter))) (doseq [f fs] (f r)))))) - +(def hall-map (ref {})) (def popular-map (ref {})) +(def score-map (ref {})) (defn update-popular [r] (if (and (:is_image r) (not (:admin_only r))) (dosync - (let [user-map (or (get @popular-map (:author r)) {}) - msg-cnt (or (get user-map (:message_id r)) 0)] - (alter popular-map assoc (:author r) - (assoc user-map (:message_id r) (inc msg-cnt))))))) + (let [author (:author r) + msg-id (:message_id r) + user-map (get @popular-map author {}) + msg-cnt (get user-map msg-id 0) + hall-cnt (get @hall-map msg-id 0) + usr-cnt (get @score-map author 0)] + (alter hall-map assoc msg-id (inc hall-cnt)) + (alter score-map assoc author (inc usr-cnt)) + (alter popular-map assoc author + (assoc user-map msg-id (inc msg-cnt))))))) (defn transmit-popular [] (doseq [[nick msgs] @popular-map] @@ -60,14 +71,30 @@ WHERE (doseq [[msg-id score] (take (* num-popular-dumps 2) sorted-msgs)] (redis/zadd userkey score msg-id))))) - ;; (redis/expire userkey (redis-days 2))))) -- TODO: schedule auto loader (println "cached popular data for" (count @popular-map) "users")) +(defn transmit-favscores [] + (redis/atomically + (redis/del "favscores") + (doseq [[nick score] @score-map] + (redis/zadd "favscores" score (lower-case nick)))) + (println "cached favscores for " (count @score-map) "users")) + +(defn transmit-hall [] + (let [scores (take (* 2 num-hall-dumps) + (sort #(>= (second %1) (second %2)) @hall-map))] + (redis/atomically + (redis/del "hall") + (doseq [[msg-id score] scores] + (redis/zadd "hall" score msg-id))))) + + (println "streaming tags") (stream-tags [update-popular]) (println (format "processed %s tags" @tag-counter)) - (redis/with-server redis-server - (transmit-popular)) + (transmit-favscores) + (transmit-popular) + (transmit-hall)) -- cgit v1.2.3-70-g09d2 From 85e6d3994f7d547fe525fb60cb8fcc0f96066c02 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Tue, 23 Nov 2010 01:24:16 -0500 Subject: Fetch directory/hall from redis --- src/config.clj | 1 + src/datalayer.clj | 68 +++++++++++++++++++++++++++++++++++++++-- src/site.clj | 90 ++++++++++++++++++++++++++++++++----------------------- src/tags.clj | 3 +- 4 files changed, 121 insertions(+), 41 deletions(-) diff --git a/src/config.clj b/src/config.clj index ce3a88c..5253379 100644 --- a/src/config.clj +++ b/src/config.clj @@ -31,3 +31,4 @@ (def *dumps-per-page* 20) (def *vip-dumps-per-page* 200) (def message-count-limit 200) +(def num-hall-dumps 50) \ No newline at end of file diff --git a/src/datalayer.clj b/src/datalayer.clj index 38a597e..0d328b6 100644 --- a/src/datalayer.clj +++ b/src/datalayer.clj @@ -2,6 +2,7 @@ (:require redis tags) (:use config + jedis utils)) @@ -28,14 +29,42 @@ WHERE u.user_id = ANY(?)" user-id) ", false AS favorited"))) +(defn recent-posts-nick-query [user-id] + (format " +SELECT u.user_id, u.nick, u.avatar, + m.content, m.message_id%s +FROM users u +LEFT JOIN messages m on m.message_id = + (SELECT message_id FROM messages + WHERE user_id = u.user_id + AND is_image + AND room_id IN (SELECT room_id from rooms where admin_only = false) + ORDER BY created_on desc LIMIT 1) +WHERE u.nick = ANY(?)" + (if user-id + (format + ", + EXISTS (SELECT 1 FROM tags + WHERE tag = 'favorite' AND user_id = %s AND message_id = m.message_id) AS favorited" + user-id) + ", false AS favorited"))) + (defn lookup-recent-posts [user-tag-id user-ids] (do-select [(recent-posts-query user-tag-id) (sql-array "int" user-ids)])) -(defn lookup-recent-posts-tagless [user-tag-id user-ids] +(defn lookup-recent-posts-tagless [user-ids] (do-select [(recent-posts-query nil) (sql-array "int" user-ids)])) +(defn lookup-recent-posts-by-nicks [user-tag-id nicks] + (do-select [(recent-posts-nick-query user-tag-id) + (sql-array "varchar" nicks)])) + +(defn lookup-recent-posts-tagless-by-nicks [nicks] + (do-select [(recent-posts-nick-query nil) + (sql-array "text" nicks)])) + (defn fetch-message-by-id [m-id] (let [query "SELECT m.message_id, m.content, m.created_on, m.user_id, u.nick, u.avatar, r.key, r.admin_only @@ -63,7 +92,7 @@ order by count desc limit ? offset ?") (defn fetch-popular-dumps [nick viewer-nick] (for [d (do-select [popular-dumps-qry nick 40 0])] - (let [favers (vec (.getArray (:user_nicks d)))] + (let [favers (.getArray (:user_nicks d))] (assoc d :favers favers :favorited (some #(= % viewer-nick) favers))))) @@ -75,3 +104,38 @@ order by count desc limit ? offset ?") msg-ids (map maybe-parse-int msg-ids)] (if-not (empty? msg-ids) (tags/fetch-dumps-by-ids msg-ids viewer-nick)))) + + +;;;; Redis Favscores + +(defn fetch-redis-directory [offset num] + (vec + (for [t (with-jedis + #(.zrevrangeWithScores % "favscores" offset (dec num)))] + {:nick (.getElement t) + :score (int (.getScore t))}))) + +(defn fetch-redis-favscore [nick] + (maybe-parse-int + (redis/with-server redis-server + (redis/zscore "favscores" (lower-case nick))) + 0)) + +(defn incrby-redis-favscore! [nick msg-id inc] + (let [msg-id (str msg-id) + inc (double inc)] + (with-jedis + #(doto % + (.zincrby "favscores" inc (lower-case nick)) + (.zincrby (str "popular:" nick) inc msg-id) + (.zincrby "hall" inc msg-id))))) + + +;;;; Redis Hall of Fame + +(defn fetch-redis-hall [viewer-nick] + (let [ids (map maybe-parse-int + (redis/with-server redis-server + (redis/zrevrange "hall" 0 (dec num-hall-dumps))))] + (if-not (empty? ids) + (tags/fetch-dumps-by-ids ids viewer-nick)))) diff --git a/src/site.clj b/src/site.clj index e79b937..3548149 100644 --- a/src/site.clj +++ b/src/site.clj @@ -376,6 +376,8 @@ ORDER BY cnt DESC (comp take-images :content) dumps)))))) +(def use-redis-favscore true) + (defn profile ([session profile-nick] (profile session profile-nick "profile")) ([session profile-nick template] @@ -385,7 +387,9 @@ ORDER BY cnt DESC nick (session :nick) logger (make-time-logger) is-home (and nick (= nick profile-nick)) - score (lookup-score profile-nick) + score (if use-redis-favscore + (fetch-redis-favscore profile-nick) + (lookup-score profile-nick)) dumps (logger tags/fetch-dumps :user-tag-id (:user_id session) :nick profile-nick @@ -448,7 +452,9 @@ ORDER BY cnt DESC (defn build-mini-profile [user-info] (let [st (fetch-template-fragment "mini_profile") nick (user-info :nick) - score (lookup-score nick)] + score (if use-redis-favscore + (fetch-redis-favscore nick) + (lookup-score nick))] (doseq [a [:nick :avatar :contact :bio]] (let [v (user-info a)] (.setAttribute st (name a) @@ -516,6 +522,7 @@ ORDER BY cnt DESC raw-dumps (if use-popular-redis (fetch-popular-dumps-redis profile-nick (:nick session)) (fetch-popular-dumps profile-nick (:nick session))) + raw-dumps (filter #(> (:count %) 0) raw-dumps) dumps (map process-message-for-output raw-dumps)] (.setAttribute st "nick" profile-nick) (.setAttribute st "mini_profile" (build-mini-profile user-info)) @@ -528,10 +535,8 @@ ORDER BY cnt DESC (def *per-directory-page* 25) (defn process-directory-entry [entry] - (let [score (lookup-score (:nick entry))] - (assoc (stringify-and-escape entry) - "score_ent" (score-to-entity score) - "score" score))) + (assoc (stringify-and-escape entry) + "score_ent" (score-to-entity (:score entry)))) (def directory-cache-ttl (minutes 10)) @@ -541,15 +546,25 @@ ORDER BY cnt DESC (defn add-recent-posts [user-id users] (if-not (empty? users) - (let [f (if user-id lookup-recent-posts lookup-recent-posts-tagless) - res (f user-id (map :user_id users))] + (let [res (if user-id + (lookup-recent-posts user-id (map :user_id users)) + (lookup-recent-posts-tagless (map :user_id users)))] (for [u users] (merge u (find-first #(= (:user_id u) (:user_id %)) res)))))) +(defn add-recent-posts-nick [user-id users] + (if-not (empty? users) + (let [nicks (map :nick users) + res (if user-id + (lookup-recent-posts-by-nicks user-id nicks) + (lookup-recent-posts-tagless-by-nicks nicks))] + (for [u users] + (merge u (find-first #(= (:nick u) (:nick %)) res)))))) + (defn get-directory-info [user-id offset] - (map process-directory-entry - (add-recent-posts user-id - (get-user-ranking offset *per-directory-page*)))) + (let [res (fetch-redis-directory offset *per-directory-page*)] + (map process-directory-entry + (add-recent-posts-nick user-id res)))) (defn directory [session offset] (let [st (fetch-template "directory" session) @@ -805,28 +820,31 @@ ORDER BY cnt DESC (try (do-insert "tags" ["user_id" "message_id" "tag"] - [(:user_id user) (msg :message_id) tag]) - (if (and (= tag "favorite") - (not (= (msg :nick) (user :nick)))) + [(:user_id user) (:message_id msg) tag]) + (when (and (= tag "favorite") + (not (= (msg :nick) (:nick user)))) + (if-not (:admin_only msg) + (incrby-redis-favscore! (:nick msg) (:message_id msg) 1)) (insert-fav-notification! (msg :nick) (user :nick) (user :avatar) (msg :content))) true ; catch error when inserting duplicate tags - (catch Exception e false))) + (catch Exception e + (do (println e) + false)))) (defn validated-add-tag [session params] (if (session :nick) (let [nick (session :nick) user-id (session :user_id) - user-admin? (session :admin-only) - msg-id (params :message_id) + msg-id (params :message_id) tag (validate-tag (params :tag)) msg (fetch-message-by-id msg-id) access (or (is-vip? session) - (not (:admin-only msg)))] + (not (:admin_only msg)))] (cond (not msg) (resp-error "NO_MSG") (not access) (resp-error "NO_MSG") (not tag) (resp-error "NO_TAG") @@ -835,10 +853,18 @@ ORDER BY cnt DESC (resp-error "TAG_EXISTS_ALREADY_OR_SOMETHING_ELSE_IS_FUCKED")))) (resp-error "NO_USER"))) -(defn remove-tag [user-id message-id tag] - (let [query "user_id = ? AND message_id = ? AND lower(tag) = ?"] - (do-delete "tags" [query user-id (maybe-parse-int message-id) (normalize-tag-for-db (.toLowerCase tag))]) - (resp-success "OK"))) +(defn remove-tag [user-id msg-id tag] + (let [query "user_id = ? AND message_id = ? AND lower(tag) = ?" + msg-id (maybe-parse-int msg-id) + tag (normalize-tag-for-db tag) + msg (fetch-message-by-id msg-id)] + (let [rows-deleted (first (do-delete "tags" [query user-id msg-id tag]))] + (if-not (zero? rows-deleted) + (do + (if-not (:admin_only msg) + (incrby-redis-favscore! (:nick msg) msg-id -1)) + (resp-success "OK")) + (resp-error "NO_TAG"))))) (defn validated-remove-tag [session params] (if (session :nick) @@ -1149,9 +1175,8 @@ ORDER BY cnt DESC (unknown-page))) (defn hall-of-fame [session] - (let [st (fetch-template "fame" session) - msgs (add-user-favs-to-msgs (poll hall-results) - (session :user_id))] + (let [st (fetch-template "fame" session) + msgs (fetch-redis-hall (:nick session))] (.setAttribute st "dumps" (map process-message-for-output msgs)) (.toString st))) @@ -1394,23 +1419,12 @@ ORDER BY cnt DESC (load-rooms!) (start! reserved-nicks) -(def server (start-server (options :port))) -(start! *active-mutes*) -; Delay the following to reduce start-load -(Thread/sleep 15000) -(start! *user-scores*) +(def server (start-server (options :port))) +(start! *active-mutes*) (start-user-flusher!) (start-session-pruner!) -(start! hall-results) - -;; Scott 2010/8/30: disable feeds to test impact on server load -;; (and see if anyone notices) -;; (if (= *server-url* "http://dump.fm") -;; (do (start! feed-downloader) -;; (start! feed-inserter))) - ;(if (not= *server-url* "http://dump.fm") ; (start! random-poster)) diff --git a/src/tags.clj b/src/tags.clj index 37c93b0..bc022f9 100644 --- a/src/tags.clj +++ b/src/tags.clj @@ -15,7 +15,8 @@ (.toLowerCase tag))) ; save all spaces in tags as dashes? -(defn normalize-tag-for-db [tag] (str tag)) +(defn normalize-tag-for-db [tag] + (lower-case tag)) ; (.replace tag " " "-")) ; todo: remove unicode escape sequences and line breaks and stuff? -- cgit v1.2.3-70-g09d2 From ad83cbd56471a628cf4cd657bdea962f4f323973 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Tue, 23 Nov 2010 01:40:43 -0500 Subject: Count txt messages for favscore --- src/redisload.clj | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/redisload.clj b/src/redisload.clj index 07c1275..31a25f0 100644 --- a/src/redisload.clj +++ b/src/redisload.clj @@ -49,16 +49,16 @@ WHERE (def score-map (ref {})) (defn update-popular [r] - (if (and (:is_image r) (not (:admin_only r))) - (dosync - (let [author (:author r) - msg-id (:message_id r) - user-map (get @popular-map author {}) - msg-cnt (get user-map msg-id 0) - hall-cnt (get @hall-map msg-id 0) - usr-cnt (get @score-map author 0)] - (alter hall-map assoc msg-id (inc hall-cnt)) - (alter score-map assoc author (inc usr-cnt)) + (dosync + (let [author (:author r) + msg-id (:message_id r) + user-map (get @popular-map author {}) + msg-cnt (get user-map msg-id 0) + hall-cnt (get @hall-map msg-id 0) + usr-cnt (get @score-map author 0)] + (alter score-map assoc author (inc usr-cnt)) + (alter hall-map assoc msg-id (inc hall-cnt)) + (if (:is_image r) (alter popular-map assoc author (assoc user-map msg-id (inc msg-cnt))))))) -- cgit v1.2.3-70-g09d2 From 048783c9e591d859bda8b51a63086e0330f6fd30 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Tue, 23 Nov 2010 01:58:52 -0500 Subject: Updated fav handling --- src/datalayer.clj | 12 +++++++----- src/site.clj | 8 ++++---- src/tags.clj | 11 ++++++----- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/datalayer.clj b/src/datalayer.clj index 0d328b6..26fbd36 100644 --- a/src/datalayer.clj +++ b/src/datalayer.clj @@ -121,14 +121,16 @@ order by count desc limit ? offset ?") (redis/zscore "favscores" (lower-case nick))) 0)) -(defn incrby-redis-favscore! [nick msg-id inc] +(defn incrby-redis-favscore! [nick msg-id inc is-image] + (println nick msg-id inc is-image) (let [msg-id (str msg-id) inc (double inc)] (with-jedis - #(doto % - (.zincrby "favscores" inc (lower-case nick)) - (.zincrby (str "popular:" nick) inc msg-id) - (.zincrby "hall" inc msg-id))))) + #(do + (.zincrby % "favscores" inc (lower-case nick)) + (when is-image + (.zincrby % (str "popular:" nick) inc msg-id) + (.zincrby % "hall" inc msg-id)))))) ;;;; Redis Hall of Fame diff --git a/src/site.clj b/src/site.clj index 3548149..3a9c379 100644 --- a/src/site.clj +++ b/src/site.clj @@ -823,8 +823,8 @@ ORDER BY cnt DESC [(:user_id user) (:message_id msg) tag]) (when (and (= tag "favorite") (not (= (msg :nick) (:nick user)))) - (if-not (:admin_only msg) - (incrby-redis-favscore! (:nick msg) (:message_id msg) 1)) + (if-not (or (:admin_only msg) (= (:user_id user) (:user_id msg))) + (incrby-redis-favscore! (:nick msg) (:message_id msg) 1 (:is_image msg))) (insert-fav-notification! (msg :nick) (user :nick) (user :avatar) @@ -861,8 +861,8 @@ ORDER BY cnt DESC (let [rows-deleted (first (do-delete "tags" [query user-id msg-id tag]))] (if-not (zero? rows-deleted) (do - (if-not (:admin_only msg) - (incrby-redis-favscore! (:nick msg) msg-id -1)) + (if-not (or (:admin_only msg) (= user-id (:user_id msg))) + (incrby-redis-favscore! (:nick msg) msg-id -1 (:is_image msg))) (resp-success "OK")) (resp-error "NO_TAG"))))) diff --git a/src/tags.clj b/src/tags.clj index bc022f9..53236c5 100644 --- a/src/tags.clj +++ b/src/tags.clj @@ -339,13 +339,14 @@ WHERE EXISTS index-func (fn [row] (index-of #(= (:message_id row) %) ids))] (for [m (sort-by index-func tagged-rows)] - (let [favers (get (:tags m) "favorite") - favorited (and viewer-nick - (boolean (some #(= % viewer-nick) favers)))] + (let [favers (get (:tags m) "favorite") + favorited (and viewer-nick + (boolean (some #(= % viewer-nick) favers))) + filtered-favers (filter #(not= % (:nick m)) favers)] (assoc m - :favers favers + :favers filtered-favers :favorited favorited - :count (count favers))))))) + :count (count filtered-favers))))))) (defn fetch-dump-by-id ([m-id] -- cgit v1.2.3-70-g09d2 From 2f7bca0861362a95ab6281a4bb230e2e4acfe1b4 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Tue, 23 Nov 2010 02:07:16 -0500 Subject: scottbot fix --- src/datalayer.clj | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/datalayer.clj b/src/datalayer.clj index 26fbd36..32afa7d 100644 --- a/src/datalayer.clj +++ b/src/datalayer.clj @@ -116,10 +116,12 @@ order by count desc limit ? offset ?") :score (int (.getScore t))}))) (defn fetch-redis-favscore [nick] - (maybe-parse-int - (redis/with-server redis-server - (redis/zscore "favscores" (lower-case nick))) - 0)) + (if (= (lower-case nick) "scottbot") + (maybe-parse-int + (redis/with-server redis-server + (redis/zscore "favscores" (lower-case nick))) + 0) + -1)) (defn incrby-redis-favscore! [nick msg-id inc is-image] (println nick msg-id inc is-image) -- cgit v1.2.3-70-g09d2 From 6edb83e01c58e5b0d959409a6d77dacc2e4ad31c Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Tue, 23 Nov 2010 02:09:30 -0500 Subject: scottbot fix --- src/datalayer.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/datalayer.clj b/src/datalayer.clj index 32afa7d..eaa997f 100644 --- a/src/datalayer.clj +++ b/src/datalayer.clj @@ -117,11 +117,11 @@ order by count desc limit ? offset ?") (defn fetch-redis-favscore [nick] (if (= (lower-case nick) "scottbot") + -1 (maybe-parse-int (redis/with-server redis-server (redis/zscore "favscores" (lower-case nick))) - 0) - -1)) + 0))) (defn incrby-redis-favscore! [nick msg-id inc is-image] (println nick msg-id inc is-image) -- cgit v1.2.3-70-g09d2 From 13b8df31190253b9b1d1b7d5bc6412e8cdafdeef Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Tue, 23 Nov 2010 02:10:44 -0500 Subject: remove print statement --- src/datalayer.clj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/datalayer.clj b/src/datalayer.clj index eaa997f..f9061b0 100644 --- a/src/datalayer.clj +++ b/src/datalayer.clj @@ -124,7 +124,6 @@ order by count desc limit ? offset ?") 0))) (defn incrby-redis-favscore! [nick msg-id inc is-image] - (println nick msg-id inc is-image) (let [msg-id (str msg-id) inc (double inc)] (with-jedis -- cgit v1.2.3-70-g09d2 From 7b7409e194dc6d21a50028ea606388caf7d375cf Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Tue, 23 Nov 2010 02:17:45 -0500 Subject: Include is_image in fetch-message-by-id --- src/datalayer.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datalayer.clj b/src/datalayer.clj index f9061b0..3d4ab2b 100644 --- a/src/datalayer.clj +++ b/src/datalayer.clj @@ -67,7 +67,7 @@ WHERE u.nick = ANY(?)" (defn fetch-message-by-id [m-id] (let [query "SELECT m.message_id, m.content, m.created_on, m.user_id, - u.nick, u.avatar, r.key, r.admin_only + m.is_image, u.nick, u.avatar, r.key, r.admin_only FROM messages m, users u, rooms r WHERE m.user_id = u.user_id AND r.room_id = m.room_id -- cgit v1.2.3-70-g09d2 From 3185c6e3bdb46f57de5e8f39f1df6338a3261398 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Wed, 24 Nov 2010 00:09:33 -0500 Subject: Remove log except for vips --- src/site.clj | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/site.clj b/src/site.clj index 3a9c379..cf6ae14 100644 --- a/src/site.clj +++ b/src/site.clj @@ -246,7 +246,7 @@ (set-cookie *login-token-key* "dummy" :expires "Thu, 01-Jan-1970 00:00:01 GMT" :domain *cookie-domain*) - (redirect-to "http://dump.fm/")]) + (redirect-to "/")]) ;; Registration @@ -759,10 +759,12 @@ ORDER BY cnt DESC (.toString st))) (defn validated-log [session room-key offset params] - (let [room-key (if (= (lower-case room-key) "www") "dumpfm" room-key)] - (if (validate-room-access room-key session) - (log session (lookup-room room-key) offset params) - (resp-error "UNKNOWN_ROOM")))) + (if (is-vip? session) + (let [room-key (if (= (lower-case room-key) "www") "dumpfm" room-key)] + (if (validate-room-access room-key session) + (log session (lookup-room room-key) offset params) + (resp-error "UNKNOWN_ROOM"))) + (redirect-to "/"))) ;; Hiscore test... redis test... @@ -1324,16 +1326,16 @@ ORDER BY cnt DESC (GET "/error/ie" (serve-template "error_ie" session)) ;; Put username routes below all others in priority - (GET "/:nick" (profile session (params :nick))) - (GET "/:nick/" (profile session (params :nick))) - (GET "/:nick/altars" (altar-log session params)) - (GET "/:nick/altars/" (altar-log session params)) - (GET "/:nick/altars/:id" (altar-log session params)) - (GET "/:nick/tag/:tag" (tagged-dumps-by-nick session params (request-url request))) + (GET "/:nick" (profile session (params :nick))) + (GET "/:nick/" (profile session (params :nick))) + (GET "/:nick/altars" (altar-log session params)) + (GET "/:nick/altars/" (altar-log session params)) + (GET "/:nick/altars/:id" (altar-log session params)) + (GET "/:nick/tag/:tag" (tagged-dumps-by-nick session params (request-url request))) (GET "/:nick/tag/:tag/:offset" (tagged-dumps-by-nick session params (request-url request))) - (GET "/:nick/favorites" (favorites-handler session (params :nick) nil nil)) - (GET "/:nick/favorites/" (favorites-handler session (params :nick) nil nil)) - (GET "/:nick/favorites/:date" (favorites-handler session (params :nick) (params :date) nil)) + (GET "/:nick/favorites" (favorites-handler session (params :nick) nil nil)) + (GET "/:nick/favorites/" (favorites-handler session (params :nick) nil nil)) + (GET "/:nick/favorites/:date" (favorites-handler session (params :nick) (params :date) nil)) (GET "/:nick/favorites/:date/" (favorites-handler session (params :nick) (params :date) nil)) (GET "/:nick/favorites/:date/:msg" (favorites-handler session (params :nick) (params :date) (params :msg))) (GET "/:nick/favs" (favorites-handler session (params :nick) nil nil)) -- cgit v1.2.3-70-g09d2 From 3b9c0e6224825ce798c988e1b7d9dbc8c9b43fec Mon Sep 17 00:00:00 2001 From: dumpfmprod Date: Wed, 24 Nov 2010 00:10:36 -0500 Subject: sostler prod commit css files --- static/css/dump.css | 6 +++--- static/css/front.css | 11 ++++------- static/form_login/front.css | 18 ++++++++---------- 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/static/css/dump.css b/static/css/dump.css index d4b7000..55e3130 100755 --- a/static/css/dump.css +++ b/static/css/dump.css @@ -140,7 +140,7 @@ left:-1px; box-shadow: 0 0 10px #d8dbde, 0px 0px 2px #000; -webkit-box-shadow: 0 0 10px #d8dbde, 0px 0px 2px #000; -moz-box-shadow: 0 0 10px #d8dbde, 0px 0px 2px #000; -z-index: 1; +z-index: 10000000; } #bar7{ top:19px; @@ -1179,7 +1179,7 @@ border-bottom:2px solid #c8cbce; filter: progid:DXImageTransform.Microsoft.dropShadow(color=#c8cbce, offX=3, offY=4, positive=true); text-overflow:ellipsis; /* opacity:0.75;*/ - z-index:18; + z-index:1000; text-align: left; background-image:url(/static/img/bg.dither.gif); background-color:#fff; @@ -1260,7 +1260,7 @@ bottom:65px; width: 99.6%; background-color:#fff; left:0.2%; - z-index:; + z-index:0; } #edit-toggle{ position:absolute; diff --git a/static/css/front.css b/static/css/front.css index 1bf973e..4adfbe0 100644 --- a/static/css/front.css +++ b/static/css/front.css @@ -5,11 +5,8 @@ right:3; color:#fff; position: relative; -top:-2; -filter:alpha(opacity=90); - -moz-opacity:0.9; - -khtml-opacity: 0.9; - opacity: 0.9z-index:2; +top:8px; +z-index:20000000; } @@ -32,7 +29,7 @@ filter:alpha(opacity=90); background-color:#fff; - +z-index:20000000; *padding:4px 12px 6px; filter:alpha(opacity=80); @@ -95,7 +92,7 @@ a.signin.menu-open span {cursor:pointer; color:#789; font-size:11px; box-shadow: 3px 2px 2px #ccc; -webkit-box-shadow: 3px 2px 2px #ccc; -z-index:5000; +z-index:50000000; -moz-box-shadow: 3px 2px 2px #ccc; filter: progid:DXImageTransform.Microsoft.dropShadow(color=#ccc, offX=3, offY=4, positive=true); } diff --git a/static/form_login/front.css b/static/form_login/front.css index 1bd0fae..6c47ab6 100644 --- a/static/form_login/front.css +++ b/static/form_login/front.css @@ -5,11 +5,8 @@ right:3; color:#fff; position: relative; -top:-2; -filter:alpha(opacity=90); - -moz-opacity:0.9; - -khtml-opacity: 0.9; - opacity: 0.9z-index:2; +top:10px; +z-index:20000000; } @@ -32,7 +29,7 @@ filter:alpha(opacity=90); background-color:#fff; - +z-index:20000000; *padding:4px 12px 6px; filter:alpha(opacity=80); @@ -52,7 +49,7 @@ cursor:pointer; margin-left:3px; } a.signin span { - background-image:url("images/toggle_down_light.png"); + background-image:url("http://dump.fm/static/form_login/images/toggle_down_light.png"); background-repeat:no-repeat; background-position:100% 50%; padding:4px 16px 6px 0;cursor:pointer; @@ -68,7 +65,7 @@ filter: progid:DXImageTransform.Microsoft.dropShadow(color=#ccc, offX=3, offY=4, } a.signin.menu-open span {cursor:pointer; - background-image:url("images/toggle_up_dark.png"); + background-image:url("http://dump.fm/static/form_login/images/toggle_up_dark.png"); color:#789; } @@ -95,7 +92,7 @@ a.signin.menu-open span {cursor:pointer; color:#789; font-size:11px; box-shadow: 3px 2px 2px #ccc; -webkit-box-shadow: 3px 2px 2px #ccc; -z-index:5000; +z-index:50000000; -moz-box-shadow: 3px 2px 2px #ccc; filter: progid:DXImageTransform.Microsoft.dropShadow(color=#ccc, offX=3, offY=4, positive=true); } @@ -125,7 +122,8 @@ filter: progid:DXImageTransform.Microsoft.dropShadow(color=#ccc, offX=3, offY=4, width:90px; height:35px; font-size:20px; - background-image:url(/static/img/btngrad1.png); + background-image:url('http://dump.fm/static/img/btngrad1.png'); + background-color:#4691e5; font-weight:bold; word-spacing:7; margin-top:-5px; -- cgit v1.2.3-70-g09d2 From afb3286c00c33db372febafb54c84701a174092e Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Fri, 26 Nov 2010 13:47:48 -0500 Subject: Fix redis directory --- src/datalayer.clj | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/datalayer.clj b/src/datalayer.clj index 3d4ab2b..139274b 100644 --- a/src/datalayer.clj +++ b/src/datalayer.clj @@ -108,10 +108,13 @@ order by count desc limit ? offset ?") ;;;; Redis Favscores -(defn fetch-redis-directory [offset num] +(defn fetch-redis-directory [page num] (vec (for [t (with-jedis - #(.zrevrangeWithScores % "favscores" offset (dec num)))] + #(.zrevrangeWithScores % + "favscores" + (* page num) + (dec (* (inc page) num))))] {:nick (.getElement t) :score (int (.getScore t))}))) -- cgit v1.2.3-70-g09d2 From dd46cb29fa939546908db15fc92491bc49f3130f Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Mon, 29 Nov 2010 01:15:49 -0500 Subject: Commit initial vip-only direct messaging --- db/0-create.psql | 10 ++-- src/config.clj | 3 +- src/datalayer.clj | 70 +++++++++++++++++++++- src/feed.clj | 2 + src/message.clj | 39 ++++++++++++ src/rooms.clj | 13 +--- src/site.clj | 94 +++++++++++++++-------------- src/user.clj | 60 +++++++++++++++++-- src/utils.clj | 23 ------- static/js/pichat.js | 158 ++++++++++++++++++++++++++++--------------------- template/head.st | 4 +- template/profile.st | 20 ++++++- template/rooms/VIP.st | 3 + template/rooms/chat.st | 3 + 14 files changed, 338 insertions(+), 164 deletions(-) create mode 100644 src/message.clj diff --git a/db/0-create.psql b/db/0-create.psql index 4fa8536..762bc3a 100644 --- a/db/0-create.psql +++ b/db/0-create.psql @@ -137,11 +137,11 @@ CREATE TABLE invalid_feed_images ( CREATE INDEX invalid_feed_images_idx ON invalid_feed_images (image_url); -CREATE TABLE events ( - event_id SERIAL PRIMARY KEY, - name text NOT NULL, - author integer NOT NULL REFERENCES, - created_on timestamp NOT NULL DEFAULT now() +CREATE TABLE direct_messages ( + dm_id SERIAL PRIMARY KEY, + message_id integer NOT NULL REFERENCES messages, + author_id integer NOT NULL REFERENCES users, + recip_id integer NOT NULL REFERENCES users ); -- dont add this yet diff --git a/src/config.clj b/src/config.clj index 5253379..c4e2fe3 100644 --- a/src/config.clj +++ b/src/config.clj @@ -31,4 +31,5 @@ (def *dumps-per-page* 20) (def *vip-dumps-per-page* 200) (def message-count-limit 200) -(def num-hall-dumps 50) \ No newline at end of file +(def num-hall-dumps 50) +(def max-content-size 2468) \ No newline at end of file diff --git a/src/datalayer.clj b/src/datalayer.clj index 139274b..28ef3bf 100644 --- a/src/datalayer.clj +++ b/src/datalayer.clj @@ -1,8 +1,13 @@ (ns datalayer (:require redis tags) - (:use config + (:use clojure.contrib.sql + clojure.contrib.json.write + clojure.contrib.json.read + config jedis + message + user utils)) @@ -74,8 +79,6 @@ WHERE u.nick = ANY(?)" AND m.message_id = ?"] (first (do-select [query (maybe-parse-int m-id -1)])))) - - ;;;; Popular Posts (def popular-dumps-qry " @@ -145,3 +148,64 @@ order by count desc limit ? offset ?") (redis/zrevrange "hall" 0 (dec num-hall-dumps))))] (if-not (empty? ids) (tags/fetch-dumps-by-ids ids viewer-nick)))) + +;;;; Message insertion + +(def msg-insert-query + "INSERT INTO messages (user_id, room_id, content, is_image, is_text) + VALUES (?, ?, ?, ?, ?) RETURNING message_id, created_on") + +(defn insert-message-into-postgres! [author-id room-id content is-image is-text recips] + (with-connection *db* + (transaction + (let [{msg-id :message_id ts :created_on} + (first + (do-select [msg-insert-query + author-id room-id content is-image is-text]))] + (doseq [r recips] + (insert-values + :direct_messages + [:message_id :author_id :recip_id] + [msg-id author-id (:user_id r)])) + [msg-id ts])))) + +(defn insert-recips-into-redis! [recips author-id ts content] + (let [dm-json (json-str {"author_id" author-id + "recips" (map :nick recips) + "content" content})] + (redis/with-server redis-server + (redis/atomically + (doseq [r recips] + (redis/zadd (str "directmessage:" (:user_id r)) + (.getTime ts) + dm-json)))))) + +(defn insert-message! [author-id author-nick room-id content] + (let [msg-type (classify-msg content) + is-image (boolean (#{:image :mixed} msg-type)) + is-text (boolean (#{:mixed :text} msg-type)) + recips (get-recips content) + [msg-id ts] (insert-message-into-postgres! author-id + room-id + content + is-image + is-text + recips)] + (if-not (empty? recips) + (insert-recips-into-redis! recips author-id ts content)) + {:author author-nick + :msg-id msg-id + :room room-id + :db-ts ts + :content content + :recips (map (comp lower-case :nick) recips)})) + +(defn fetch-private-messages [user-id] + (for [dm (redis/with-server redis-server + (redis/zrevrange (str "directmessage:" user-id) 0 40))] + (let [dm (read-json dm) + info (fetch-user-id (get dm "author_id"))] + {"nick" (:nick info) + "content" (get dm "content") + "recips" (get dm "recips") + "avatar" (:avatar info)}))) diff --git a/src/feed.clj b/src/feed.clj index c8454d0..898f085 100755 --- a/src/feed.clj +++ b/src/feed.clj @@ -16,6 +16,8 @@ scheduled-agent utils)) +;; DEPRECATED + (def *feeds-path* "docs/feeds.csv") (defn queue-image! [room-key img] diff --git a/src/message.clj b/src/message.clj new file mode 100644 index 0000000..a8e0e9b --- /dev/null +++ b/src/message.clj @@ -0,0 +1,39 @@ +(ns message + (:use user)) + +;; Message parsing + +;; http://snippets.dzone.com/posts/show/6995 +(def url-regex #"(?i)^((http\:\/\/|https\:\/\/|ftp\:\/\/)|(www\.))+(\w+:{0,1}\w*@)?(\S+)(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?$") +(def pic-regex #"(?i)\.(jpg|jpeg|png|gif|bmp|svg)(\?|&|$)") + +(defn is-image? [word] + (and (re-find url-regex word) + (re-find pic-regex word))) + +(defn take-images [content] + (filter is-image? (.split content " "))) + +(defn classify-msg [msg] + (let [words (.split msg " ") + imgs (map is-image? words)] + (cond (every? boolean imgs) :image + (some boolean imgs) :mixed + :else :text))) + +(def recip-regex #"(?:^|\s)@\w+") + +(defn get-recips [content] + (filter + boolean + (for [at-nick (re-seq recip-regex content)] + (fetch-nick (.substring (.trim at-nick) 1))))) + +(defn get-recips-from-msgs [msgs] + (let [recips (set (apply concat + (for [m msgs] + (re-seq recip-regex (:content m)))))] + (filter + boolean + (for [r recips] + (fetch-nick (.substring (.trim r) 1)))))) diff --git a/src/rooms.clj b/src/rooms.clj index 574532a..3367ea9 100644 --- a/src/rooms.clj +++ b/src/rooms.clj @@ -6,7 +6,7 @@ utils user)) -(defstruct message-struct :nick :content :created_on :msg_id) +(defstruct message-struct :nick :content :created_on :msg_id :recips) (def *run-flusher* true) (def *flusher-sleep* (seconds 4)) @@ -99,19 +99,12 @@ ; Note: To ensure that the msg's timestamp is consistent ; with other msg creations, build-msg must be used ; within a dosync. -(defn build-msg [nick content msg-id] - (struct message-struct nick content (new Date) msg-id)) +(defn build-msg [nick content msg-id recips] + (struct message-struct nick content (new Date) msg-id recips)) (defn add-message [msg room] (insert-and-truncate! (room :messages) msg message-count-limit)) -(defn insert-message-into-db! [user-id room-id content is-image] - (:message_id - (first - (do-select ["INSERT INTO messages (user_id, room_id, content, is_image) - VALUES (?, ?, ?, ?) RETURNING message_id" - user-id room-id content is-image])))) - (defn create-and-add-room! [key] (do-select ["INSERT INTO rooms (key, name, description) VALUES (?, ?, ?) RETURNING room_id" diff --git a/src/site.clj b/src/site.clj index cf6ae14..7db43e7 100644 --- a/src/site.clj +++ b/src/site.clj @@ -21,10 +21,10 @@ datalayer email fame + message utils cookie-login session-sweeper - feed rooms tags scheduled-agent @@ -119,14 +119,13 @@ ;; User-id/nick cache ;; I keep needing to grab user-id from a nick so I thought I'd cache them -;; @timb: I just duplicated this in the user-info map :( -;; we should reconcile our user caches -(def user-id-cache (ref {})) +;; sostler todo: will replace this w/ user/user-id-cache soon +(def *user-id-cache* (ref {})) (def *user-id-cache-size* 500) (defn user-id-from-nick [nick] (let [nick (lower-case nick) - found (@user-id-cache nick)] + found (@*user-id-cache* nick)] (if found found (let [query (str "SELECT user_id FROM users WHERE lower(nick) = ?") @@ -135,8 +134,8 @@ nil (let [found (res :user_id)] (dosync - (if (> (count @user-id-cache) *user-id-cache-size*) (ref-set user-id-cache {})) - (alter user-id-cache assoc nick found)) + (if (> (count @*user-id-cache*) *user-id-cache-size*) (ref-set *user-id-cache* {})) + (alter *user-id-cache* assoc nick found)) found)))))) ;; Login code @@ -384,7 +383,7 @@ ORDER BY cnt DESC (if-let [user-info (fetch-nick profile-nick)] (let [st (fetch-template template session) profile-nick (:nick user-info) ; Update to get right casing - nick (session :nick) + nick (:nick session) logger (make-time-logger) is-home (and nick (= nick profile-nick)) score (if use-redis-favscore @@ -394,6 +393,10 @@ ORDER BY cnt DESC :user-tag-id (:user_id session) :nick profile-nick :limit 10) + dms (if-vip + (fetch-private-messages (:user_id user-info))) + recips (if dms + (set (concat (map #(get % "recips") dms)))) imgs (pull-random-dump-images dumps 5)] (do (.setAttribute st "is_home" is-home) @@ -403,6 +406,9 @@ ORDER BY cnt DESC (if (non-empty-string? v) (escape-html v))))) (.setAttribute st "score" (comma-format score)) (.setAttribute st "score_ent" (score-to-entity score)) + (when-not (empty? dms) + (.setAttribute st "dms" dms) + (.setAttribute st "recips" (json-str (map lower-case recips)))) (if (not (empty? imgs)) (.setAttribute st "imgs" imgs)) (.setAttribute st "debug_log_items" (logger)) @@ -411,7 +417,8 @@ ORDER BY cnt DESC (defn update-user-db [user-id attr val] (with-connection *db* - (update-values "users" ["user_id = ?" user-id] {attr val}))) + (update-values "users" ["user_id = ?" user-id] {attr val})) + (update-cache! user-id attr val)) (defn update-avatar [session url] (update-user-db (session :user_id) "avatar" url) @@ -614,6 +621,7 @@ ORDER BY cnt DESC :user-tag-id (:user_id session) :hide-vip false :limit (:history_size room))) + recips (map :nick (get-recips-from-msgs raw-msgs)) message-list (to-array (map process-message-for-output raw-msgs))] (if nick (dosync @@ -621,6 +629,7 @@ ORDER BY cnt DESC (doto st (.setAttribute "users" (prepare-user-list room true)) (.setAttribute "messages" message-list) + (.setAttribute "recips" (json-str (map lower-case recips))) (.setAttribute "roomkey" (room :key)) (.setAttribute "isadminroom" (room :admin_only)) (.setAttribute "json_room_key" (json-str (room :key))) @@ -670,34 +679,29 @@ ORDER BY cnt DESC (str "" content "")) (str content))) -(defn msg-db [user-id room-id content] - (let [msg-type (classify-msg content) - is-image (boolean (#{:image :mixed} msg-type)) - is-text (boolean (#{:mixed :text} msg-type)) - qry (str "INSERT INTO messages (user_id, room_id, content, is_image, is_text) " - "VALUES (?, ?, ?, ?, ?) RETURNING message_id")] - (with-connection *db* - ((first (do-select [qry user-id room-id content is-image is-text])) - :message_id)))) - (defn msg [session params] - (let [user-id (session :user_id) - mute (get (poll *active-mutes*) user-id) - nick (session :nick) - room-key (params :room) - room (lookup-room room-key) - content (.trim (params :content))] - (cond (not room) (resp-error "BAD_ROOM") - (not nick) (resp-error "NOT_LOGGED_IN") - mute (resp-error (format-mute mute)) + (let [user-id (session :user_id) + mute (get (poll *active-mutes*) user-id) + nick (session :nick) + room-key (params :room) + room (lookup-room room-key) + content (.trim (params :content)) + content-too-long? (> (count content) + max-content-size)] + (cond (not room) (resp-error "BAD_ROOM") + (not nick) (resp-error "NOT_LOGGED_IN") + content-too-long? (resp-error "TOO_LONG") + mute (resp-error (format-mute mute)) :else - (let [content (validated-content content session) - msg-id (msg-db user-id (room :room_id) content)] + (let [content (validated-content content session) + msg-info (insert-message! user-id nick (:room_id room) content) + msg-id (:msg-id msg-info)] (dosync (if (not (contains? (ensure (room :users)) nick)) (login-user (user-struct-from-session session) room)) - (add-message (build-msg nick content msg-id) room)) - (resp-success msg-id))))) + (add-message (build-msg nick content msg-id (:recips msg-info)) room)) + (resp-success {:msgid msg-id + :recips (:recips msg-info)}))))) (defn validated-msg [session params request] @@ -737,7 +741,8 @@ ORDER BY cnt DESC dump-offset (* offset *dumps-per-page*) image-only (and (not (room :admin_only)) (not= (params :show) "all")) - raw-dumps (logger tags/fetch-dumps-by-room :room-id (room :room_id) + raw-dumps (logger tags/fetch-dumps-by-room + :room-id (room :room_id) :image-only image-only :amount (+ 1 *dumps-per-page*) :offset dump-offset) @@ -1113,21 +1118,20 @@ ORDER BY cnt DESC ; errors. ; The upload code doesn't use jQuery.ajax, and doesn't JSON-eval ; responses. Therefore, return strings should not be JSON-encoded. - (defn do-upload [session image room] (if-let [err (validate-upload-file (image :tempfile) room)] (resp-error err) - (let [filename (format-filename (:filename image) (session :nick)) - date (today) - dest (open-file [*image-directory* date] filename) - url (image-url-from-file "images" date dest) - msg-id (msg-db (session :user_id) (room :room_id) url) - msg (struct message-struct (session :nick) url (new Date) msg-id)] - (do - (dosync - (add-message msg room)) - (copy (:tempfile image) dest) - [200 "OK"])))) + (let [filename (format-filename (:filename image) (session :nick)) + date (today) + dest (open-file [*image-directory* date] filename) + url (image-url-from-file "images" date dest) + msg-info (insert-message! (:user_id session) (:nick session) + (:room_id room) url)] + (copy (:tempfile image) dest) + (dosync + (let [msg (build-msg (:nick session) url (:msg-id msg-info) (:recips msg-info))] + (add-message msg room))) + [200 "OK"]))) (defn upload [session params request] (let [room-key (params :room) diff --git a/src/user.clj b/src/user.clj index 1d59944..7641bd8 100644 --- a/src/user.clj +++ b/src/user.clj @@ -16,12 +16,62 @@ (> (count n) 16) "NICK_TOO_LONG" (not (re-matches *nick-regex* n)) "NICK_INVALID_CHARS")) +;;; User info cache + +(def user-cache-size 500) +(def user-nick-cache (ref {})) +(def user-id-cache (ref {})) + +(defn update-cache! [uid attr val] + (dosync + (if-let [info (get @user-id-cache uid)] + (let [nick (lower-case (:nick info)) + new-info (assoc info attr val)] + (alter user-id-cache assoc uid new-info) + (alter user-nick-cache assoc nick new-info))))) + + (defn fetch-nick [nick] - (let [q1 "SELECT * FROM users WHERE nick = ? LIMIT 1" - ; ORDER BY ensures consistent retrieval of ambiguious names - q2 "SELECT * FROM users WHERE lower(nick) = ? ORDER BY nick LIMIT 1"] - (or (first-or-nil (do-select [q1 nick])) - (first-or-nil (do-select [q2 (lower-case nick)]))))) + (let [lcnick (lower-case nick)] + (if (contains? user-nick-cache lcnick) + (get user-nick-cache lcnick) + (let [info (first + (do-select ["SELECT * FROM users WHERE lower(nick) = ? LIMIT 1" + lcnick])) + user-id (:user_id info)] + (dosync + (alter user-nick-cache assoc lcnick info) + (if (and info user-id) + (alter user-id-cache assoc user-id info))) + info)))) + +(defn fetch-nicks [nicks] + (let [lcnicks (map lower-case nicks) + cache @user-nick-cache + to-fetch (filter #(not (contains? cache %)) lcnicks) + fetched-info (do-select ["SELECT * FROM users WHERE lower(nick) = ANY(?)" + (sql-array "text" to-fetch)]) + info-map (zipmap (map (comp lower-case :nick) fetched-info) + fetched-info)] + (doseq [nick to-fetch] + (let [info (get info-map nick)] + (dosync + (alter user-nick-cache assoc nick info) + (if info + (alter user-id-cache assoc (:user_id info) info))))) + (filter + boolean + (for [nick lcnicks] + (get @user-nick-cache nick))))) + +(defn fetch-user-id [uid] + (if (contains? @user-id-cache uid) + (get @user-id-cache uid) + (if-let [info (first + (do-select ["SELECT * FROM users WHERE user_id = ? LIMIT 1" uid]))] + (dosync + (alter user-nick-cache assoc (lower-case (:nick info)) info) + (alter user-id-cache assoc uid info))))) (defn authorize-nick-hash [nick hash] (let [db-user (fetch-nick nick)] diff --git a/src/utils.clj b/src/utils.clj index 84454cd..8aaffba 100755 --- a/src/utils.clj +++ b/src/utils.clj @@ -36,26 +36,6 @@ (.setMaxConnections 10))})) -;; Message parsing - -;; http://snippets.dzone.com/posts/show/6995 -(def url-regex #"(?i)^((http\:\/\/|https\:\/\/|ftp\:\/\/)|(www\.))+(\w+:{0,1}\w*@)?(\S+)(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?$") -(def pic-regex #"(?i)\.(jpg|jpeg|png|gif|bmp|svg)(\?|&|$)") - -(defn is-image? [word] - (and (re-find url-regex word) - (re-find pic-regex word))) - -(defn take-images [content] - (filter is-image? (.split content " "))) - -(defn classify-msg [msg] - (let [words (.split msg " ") - imgs (map is-image? words)] - (cond (every? boolean imgs) :image - (some boolean imgs) :mixed - :else :text))) - ;; Misc (defn except! [& more] @@ -415,9 +395,6 @@ (defn serve-template [template session] (.toString (fetch-template template session))) -(defn first-or-nil [l] - (if (empty? l) nil (first l))) - ;; VIP (defn is-vip? [session] diff --git a/static/js/pichat.js b/static/js/pichat.js index 6a6d962..0a849eb 100644 --- a/static/js/pichat.js +++ b/static/js/pichat.js @@ -119,6 +119,8 @@ Log.initialize(); URLRegex = /((\b(http\:\/\/|https\:\/\/|ftp\:\/\/)|(www\.))+(\w+:{0,1}\w*@)?(\S+)(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?)/gi; PicRegex = /\.(jpg|jpeg|png|gif|bmp|svg|fid)$/i; +RecipRegex = /(^|\s)@\w+/g; + function getImagesAsArray(text) { var imgs = [] @@ -133,10 +135,24 @@ function getImagesAsArray(text) { return imgs } -function linkify(text) { - LastMsgContainsImage = false - text = text.replace(URLRegex, linkReplace); - return text +function linkify(text, recips) { + LastMsgContainsImage = false; + var recipWrapper = function(text) { return recipientReplace(text, recips); }; + return text.replace(URLRegex, linkReplace).replace(RecipRegex, recipWrapper); +} + +function recipientReplace(atText, recips) { + if (atText[0] == ' ') { + atText = atText.slice(1); + var space = ' '; + } else { + var space = ''; + } + var nick = atText.slice(1); + if (!recips || recips.indexOf(nick.toLowerCase()) == -1) { + return space + atText; + } else + return space + '' + atText + ''; } // use this in escapeHtml to turn everyone's text lIkE tHiS @@ -312,22 +328,23 @@ function setImgsEnable() { } }; -function buildMsgContent(content) { +function buildMsgContent(content, recips) { if (content.substr(0,6) == "") return content.substr(6,content.length - 13) - else return linkify(escapeHtml(content)); + else return linkify(escapeHtml(content), recips); } // todo: // isLoading doesn't get passed the right thing by $.map in addMessages -function buildMessageDiv(msg, isLoading) { +function buildMessageDiv(msg, opts) { + var opts = opts || {}; var nick = escapeHtml(msg.nick); removeOldMessages(); - var builtContent = buildMsgContent(msg.content); + var builtContent = buildMsgContent(msg.content, opts.recips); var msgId = ('msg_id' in msg) ? 'id="message-' + msg.msg_id + '"' : ''; - var loadingClass = isLoading ? ' loading' : ''; + var loadingClass = opts.isLoading ? ' loading' : ''; var containsImageClass = LastMsgContainsImage ? ' contains-image' : ''; var displayStyle = ((ImgsEnabled && LastMsgContainsImage) || (TextEnabled && !LastMsgContainsImage)) ? '' : ' style="display: none"'; @@ -436,55 +453,56 @@ function clearMessages(){ } function submitMessage() { - var content = $.trim($('#msgInput').val()); - - if (content == "/clear") { - clearMessages() + var content = $.trim($('#msgInput').val()); + + if (content == "/clear") { + clearMessages() + $('#msgInput').val(''); + return; + } + + var invalidDomain = invalidImageDomain(content); + if (invalidDomain) { + $('#msgInput').blur(); // Remove focus to prevent FF alert loop + alert("Sorry, cannot accept images from " + invalidDomain + ". Maybe host the image elsewhere?"); + return; + } + $('#msgInput').val(''); - return; - } - - var invalidDomain = invalidImageDomain(content); - if (invalidDomain) { - $('#msgInput').blur(); // Remove focus to prevent FF alert loop - alert("Sorry, cannot accept images from " + invalidDomain + ". Maybe host the image elsewhere?"); - return; - } - - $('#msgInput').val(''); - if (content == '') { return; } - if (content.length > 2468) { - alert("POST TOO LONG DUDE!"); - return; - } // this shouldn't just be client side :V - PendingMessages[content] = true; - - var msg = { 'nick': Nick, 'content': content }; - var div = addNewMessage(msg, true); - - var onSuccess = function(json) { - if (typeof pageTracker !== 'undefined') { - pageTracker._trackEvent('Message', 'Submit', - typeof Room !== 'undefined' ? Room : 'UnknownRoom'); + if (content == '') { return; } + if (content.length > 2468) { + alert("POST TOO LONG DUDE!"); + return; } - div.attr('id', 'message-' + json) - .removeClass('loading').addClass('loaded'); - }; - var onError = function(resp, textStatus, errorThrown) { - div.remove(); - handleMsgError(resp); - }; - - $.ajax({ - type: 'POST', - timeout: 15000, - url: '/msg', - data: { 'room': Room, 'content': content }, - cache: false, - dataType: 'json', - success: onSuccess, - error: onError - }); + PendingMessages[content] = true; + + var msg = { 'nick': Nick, 'content': content }; + var div = addNewMessage(msg, true); + + var onSuccess = function(json) { + if (typeof pageTracker !== 'undefined') { + pageTracker._trackEvent('Message', 'Submit', + typeof Room !== 'undefined' ? Room : 'UnknownRoom'); + } + div.attr('id', 'message-' + json.msgid) + .removeClass('loading').addClass('loaded'); + div.find('.content').html(buildMsgContent(content, json.recips)); + }; + var onError = function(resp, textStatus, errorThrown) { + div.remove(); + handleMsgError(resp); + }; + + $.ajax({ + type: 'POST', + timeout: 15000, + url: '/msg', + data: { 'room': Room, 'content': content }, + cache: false, + dataType: 'json', + success: onSuccess, + error: onError + }); } function ifEnter(fn) { @@ -493,13 +511,14 @@ function ifEnter(fn) { }; } -function addNewMessages(msgs) { - var msgStr = $.map(msgs, buildMessageDiv).join(''); +function addNewMessages(msgs, recips) { + var msgOpts = { recips: recips }; + var msgStr = $.map(msgs, function(msg) { buildMessageDiv(msg, msgOpts).join(''); }); $('#messageList').append(msgStr); } function addNewMessage(msg, isLoading) { - var msgStr = buildMessageDiv(msg, isLoading); + var msgStr = buildMessageDiv(msg, { isLoading: true }); var div = $(msgStr).appendTo('#messageList'); return div; } @@ -516,11 +535,11 @@ function flattenUserJson(users) { return s; } -function updateUI(msgs, users, favs) { +function updateUI(msgs, users, favs, recips) { if (window['growlize'] && msgs && msgs.length > 0) { $.map(msgs, buildGrowlDataAndPopDatShit) } else if (msgs && msgs.length > 0) { - addNewMessages(msgs); + addNewMessages(msgs, recips); } if (users !== null) { var flattened = flattenUserJson(users); @@ -552,14 +571,13 @@ function isDuplicateMessage(m) { function refresh() { var onSuccess = function(json) { try { - Timestamp = json.timestamp; - + Timestamp = json.timestamp; $.map(json.messages, function(msg){ MessageContentCache[msg.msg_id.toString()] = msg.content }) var messages = $.grep( json.messages, function(m) { return !isDuplicateMessage(m) }); - updateUI(messages, json.users, json.favs); + updateUI(messages, json.users, json.favs, json.recips); if (!Away.HasFocus) Away.UnseenMsgCounter += messages.length; } catch(e) { @@ -619,7 +637,7 @@ function initChat() { var dump = $(this); var content = dump.find(".content") MessageContentCache[dump.attr("id").substr(8)] = content.text() - content.html(buildMsgContent(content.text())); + content.html(buildMsgContent(content.text(), Recips)); if ((ImgsEnabled && dump.hasClass('contains-image')) || (TextEnabled && !dump.hasClass('contains-image'))) dump.show(); @@ -698,11 +716,15 @@ function enableProfileEdit() { } function initProfile() { - Search.initInpage() - $(".linkify").each(function() { + Search.initInpage(); + $(".linkify-text").each(function() { var text = jQuery(this).text(); jQuery(this).html(linkifyWithoutImage(text)); - }); + }); + + $(".linkify-full").each(function() { + $(this).html(buildMsgContent($(this).text(), Recips)); + }); $('#edit-toggle').click(enableProfileEdit); activateProfileEditable(); diff --git a/template/head.st b/template/head.st index 66be7c1..a3ff733 100644 --- a/template/head.st +++ b/template/head.st @@ -1,10 +1,10 @@ - + - + diff --git a/template/profile.st b/template/profile.st index f66069e..6f44058 100644 --- a/template/profile.st +++ b/template/profile.st @@ -50,14 +50,14 @@

contact info

$if(contact)$ -
$contact$
+
$contact$
$else$ $endif$

bio

$if(bio)$ -
$bio$
+
$bio$
$else$ $endif$ @@ -90,7 +90,23 @@ style="border:none; width:500px; height:30px">
+
+ $if(dms)$ +
+
+

Messages

+ $dms: { dm | +
+ $dm.nick$ + + $dm.content$
+ + }$ +
+ $endif$
diff --git a/template/rooms/VIP.st b/template/rooms/VIP.st index f078d0d..fa38eb5 100644 --- a/template/rooms/VIP.st +++ b/template/rooms/VIP.st @@ -110,6 +110,9 @@
$messagepane()$ +
diff --git a/template/rooms/chat.st b/template/rooms/chat.st index b96456a..f7ca888 100644 --- a/template/rooms/chat.st +++ b/template/rooms/chat.st @@ -110,6 +110,9 @@
$messagepane()$ +
-- cgit v1.2.3-70-g09d2 From 7eb9104e5b0525dc2ad8c41c42577c5f9efcb910 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Mon, 29 Nov 2010 01:35:54 -0500 Subject: Correct recipient merging --- src/datalayer.clj | 2 +- src/site.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/datalayer.clj b/src/datalayer.clj index 28ef3bf..6b2a466 100644 --- a/src/datalayer.clj +++ b/src/datalayer.clj @@ -207,5 +207,5 @@ order by count desc limit ? offset ?") info (fetch-user-id (get dm "author_id"))] {"nick" (:nick info) "content" (get dm "content") - "recips" (get dm "recips") + "recips" (get dm "recips" []) "avatar" (:avatar info)}))) diff --git a/src/site.clj b/src/site.clj index 7db43e7..67fc8d4 100644 --- a/src/site.clj +++ b/src/site.clj @@ -396,7 +396,7 @@ ORDER BY cnt DESC dms (if-vip (fetch-private-messages (:user_id user-info))) recips (if dms - (set (concat (map #(get % "recips") dms)))) + (set (apply concat (map #(get % "recips") dms)))) imgs (pull-random-dump-images dumps 5)] (do (.setAttribute st "is_home" is-home) -- cgit v1.2.3-70-g09d2 From 25f6bb92ef2920d4d97fcf443573f0b09d3bb14f Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Mon, 29 Nov 2010 01:43:58 -0500 Subject: Load messages for all users, only show for vips --- src/site.clj | 6 ++---- template/profile.st | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/site.clj b/src/site.clj index 67fc8d4..fc7d541 100644 --- a/src/site.clj +++ b/src/site.clj @@ -393,10 +393,8 @@ ORDER BY cnt DESC :user-tag-id (:user_id session) :nick profile-nick :limit 10) - dms (if-vip - (fetch-private-messages (:user_id user-info))) - recips (if dms - (set (apply concat (map #(get % "recips") dms)))) + dms (fetch-private-messages (:user_id user-info)) + recips (set (apply concat (map #(get % "recips") dms))) imgs (pull-random-dump-images dumps 5)] (do (.setAttribute st "is_home" is-home) diff --git a/template/profile.st b/template/profile.st index 6f44058..ce0a9e6 100644 --- a/template/profile.st +++ b/template/profile.st @@ -91,14 +91,14 @@
- $if(dms)$ + $if(isadmin)$

Messages

$dms: { dm |
$dm.nick$ - + $dm.content$
}$
+ $endif$
-- cgit v1.2.3-70-g09d2