From e3a5d9b44762e32359a9652a6607fa7e56c874d7 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Sun, 14 Nov 2010 12:55:28 -0800 Subject: Updated config w/ redis info --- src/config.clj | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) (limited to 'src/config.clj') diff --git a/src/config.clj b/src/config.clj index 3230c72..fdb33ed 100644 --- a/src/config.clj +++ b/src/config.clj @@ -11,10 +11,20 @@ "http://localhost:8080")) (def *cookie-domain* - (if (= *server-user* "timb") - "" - ".dump.fm")) + (if (= *server-user* "dumpfmprod") + ".dump.fm" + "")) + +(def redis-server + (if (= *server-user* "dumpfmprod") + {:host "192.168.156.111" :port 6379 :db 0 } + {:host "127.0.0.1" :port 6379 :db 0 })) (def *root-directory* (System/getProperty "user.dir")) (def *image-directory* "images") -(def *avatar-directory* "avatars") \ No newline at end of file +(def *avatar-directory* "avatars") + + +;; Numerical constants + +(def num-popular-dumps 40) -- cgit v1.2.3-70-g09d2 From c37bcf6d26abf3fdadd41deda24b020b71627630 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Mon, 15 Nov 2010 01:54:43 -0800 Subject: Moved constants to config.clj --- src/config.clj | 3 +++ src/rooms.clj | 2 -- src/site.clj | 2 +- src/utils.clj | 3 --- 4 files changed, 4 insertions(+), 6 deletions(-) (limited to 'src/config.clj') diff --git a/src/config.clj b/src/config.clj index fdb33ed..7a4a6a2 100644 --- a/src/config.clj +++ b/src/config.clj @@ -28,3 +28,6 @@ ;; Numerical constants (def num-popular-dumps 40) +(def *dumps-per-page* 20) +(def *vip-dumps-per-page* 200) +(def message-count-limit 200) diff --git a/src/rooms.clj b/src/rooms.clj index e919557..7745630 100644 --- a/src/rooms.clj +++ b/src/rooms.clj @@ -101,8 +101,6 @@ (defn build-msg [nick content msg-id] (struct message-struct nick content (new Date) msg-id)) -(def message-count-limit 200) - (defn add-message [msg room] (insert-and-truncate! (room :messages) msg message-count-limit)) diff --git a/src/site.clj b/src/site.clj index 4c3560e..5d7f9a3 100644 --- a/src/site.clj +++ b/src/site.clj @@ -677,7 +677,7 @@ ORDER BY cnt DESC mute (resp-error (format-mute mute)) :else (let [content (validated-content content session) - msg-id (msg-db user-id (room :room_id) content)] + msg-id (msg-db user-id (room :room_id) content)] (dosync (if (not (contains? (ensure (room :users)) nick)) (login-user (user-struct-from-session session) room)) diff --git a/src/utils.clj b/src/utils.clj index 9d7fd3a..84454cd 100755 --- a/src/utils.clj +++ b/src/utils.clj @@ -35,9 +35,6 @@ (.setPassword db-pass) (.setMaxConnections 10))})) -;; moved this to here which doesn't seem right... maybe a 'settings.clj' or something? -(def *dumps-per-page* 20) -(def *vip-dumps-per-page* 200) ;; Message parsing -- cgit v1.2.3-70-g09d2 From b31b524d42cb2cb602b8e8c52e7c27a968770cc4 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Wed, 17 Nov 2010 01:12:29 -0500 Subject: use config.clj in rooms/tags --- src/config.clj | 2 +- src/rooms.clj | 1 + src/tags.clj | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) (limited to 'src/config.clj') diff --git a/src/config.clj b/src/config.clj index 7a4a6a2..ce3a88c 100644 --- a/src/config.clj +++ b/src/config.clj @@ -25,7 +25,7 @@ (def *avatar-directory* "avatars") -;; Numerical constants +;; Settings (def num-popular-dumps 40) (def *dumps-per-page* 20) diff --git a/src/rooms.clj b/src/rooms.clj index 7745630..9855a59 100644 --- a/src/rooms.clj +++ b/src/rooms.clj @@ -2,6 +2,7 @@ (:import java.util.Date) (:use clojure.contrib.str-utils clojure.contrib.def + config utils user)) diff --git a/src/tags.clj b/src/tags.clj index a8c3341..37c93b0 100644 --- a/src/tags.clj +++ b/src/tags.clj @@ -6,6 +6,7 @@ clojure.contrib.fcase clojure.contrib.json.write clojure.contrib.str-utils + config compojure utils)) -- cgit v1.2.3-70-g09d2 From 85e6d3994f7d547fe525fb60cb8fcc0f96066c02 Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Tue, 23 Nov 2010 01:24:16 -0500 Subject: Fetch directory/hall from redis --- src/config.clj | 1 + src/datalayer.clj | 68 +++++++++++++++++++++++++++++++++++++++-- src/site.clj | 90 ++++++++++++++++++++++++++++++++----------------------- src/tags.clj | 3 +- 4 files changed, 121 insertions(+), 41 deletions(-) (limited to 'src/config.clj') diff --git a/src/config.clj b/src/config.clj index ce3a88c..5253379 100644 --- a/src/config.clj +++ b/src/config.clj @@ -31,3 +31,4 @@ (def *dumps-per-page* 20) (def *vip-dumps-per-page* 200) (def message-count-limit 200) +(def num-hall-dumps 50) \ No newline at end of file diff --git a/src/datalayer.clj b/src/datalayer.clj index 38a597e..0d328b6 100644 --- a/src/datalayer.clj +++ b/src/datalayer.clj @@ -2,6 +2,7 @@ (:require redis tags) (:use config + jedis utils)) @@ -28,14 +29,42 @@ WHERE u.user_id = ANY(?)" user-id) ", false AS favorited"))) +(defn recent-posts-nick-query [user-id] + (format " +SELECT u.user_id, u.nick, u.avatar, + m.content, m.message_id%s +FROM users u +LEFT JOIN messages m on m.message_id = + (SELECT message_id FROM messages + WHERE user_id = u.user_id + AND is_image + AND room_id IN (SELECT room_id from rooms where admin_only = false) + ORDER BY created_on desc LIMIT 1) +WHERE u.nick = ANY(?)" + (if user-id + (format + ", + EXISTS (SELECT 1 FROM tags + WHERE tag = 'favorite' AND user_id = %s AND message_id = m.message_id) AS favorited" + user-id) + ", false AS favorited"))) + (defn lookup-recent-posts [user-tag-id user-ids] (do-select [(recent-posts-query user-tag-id) (sql-array "int" user-ids)])) -(defn lookup-recent-posts-tagless [user-tag-id user-ids] +(defn lookup-recent-posts-tagless [user-ids] (do-select [(recent-posts-query nil) (sql-array "int" user-ids)])) +(defn lookup-recent-posts-by-nicks [user-tag-id nicks] + (do-select [(recent-posts-nick-query user-tag-id) + (sql-array "varchar" nicks)])) + +(defn lookup-recent-posts-tagless-by-nicks [nicks] + (do-select [(recent-posts-nick-query nil) + (sql-array "text" nicks)])) + (defn fetch-message-by-id [m-id] (let [query "SELECT m.message_id, m.content, m.created_on, m.user_id, u.nick, u.avatar, r.key, r.admin_only @@ -63,7 +92,7 @@ order by count desc limit ? offset ?") (defn fetch-popular-dumps [nick viewer-nick] (for [d (do-select [popular-dumps-qry nick 40 0])] - (let [favers (vec (.getArray (:user_nicks d)))] + (let [favers (.getArray (:user_nicks d))] (assoc d :favers favers :favorited (some #(= % viewer-nick) favers))))) @@ -75,3 +104,38 @@ order by count desc limit ? offset ?") msg-ids (map maybe-parse-int msg-ids)] (if-not (empty? msg-ids) (tags/fetch-dumps-by-ids msg-ids viewer-nick)))) + + +;;;; Redis Favscores + +(defn fetch-redis-directory [offset num] + (vec + (for [t (with-jedis + #(.zrevrangeWithScores % "favscores" offset (dec num)))] + {:nick (.getElement t) + :score (int (.getScore t))}))) + +(defn fetch-redis-favscore [nick] + (maybe-parse-int + (redis/with-server redis-server + (redis/zscore "favscores" (lower-case nick))) + 0)) + +(defn incrby-redis-favscore! [nick msg-id inc] + (let [msg-id (str msg-id) + inc (double inc)] + (with-jedis + #(doto % + (.zincrby "favscores" inc (lower-case nick)) + (.zincrby (str "popular:" nick) inc msg-id) + (.zincrby "hall" inc msg-id))))) + + +;;;; Redis Hall of Fame + +(defn fetch-redis-hall [viewer-nick] + (let [ids (map maybe-parse-int + (redis/with-server redis-server + (redis/zrevrange "hall" 0 (dec num-hall-dumps))))] + (if-not (empty? ids) + (tags/fetch-dumps-by-ids ids viewer-nick)))) diff --git a/src/site.clj b/src/site.clj index e79b937..3548149 100644 --- a/src/site.clj +++ b/src/site.clj @@ -376,6 +376,8 @@ ORDER BY cnt DESC (comp take-images :content) dumps)))))) +(def use-redis-favscore true) + (defn profile ([session profile-nick] (profile session profile-nick "profile")) ([session profile-nick template] @@ -385,7 +387,9 @@ ORDER BY cnt DESC nick (session :nick) logger (make-time-logger) is-home (and nick (= nick profile-nick)) - score (lookup-score profile-nick) + score (if use-redis-favscore + (fetch-redis-favscore profile-nick) + (lookup-score profile-nick)) dumps (logger tags/fetch-dumps :user-tag-id (:user_id session) :nick profile-nick @@ -448,7 +452,9 @@ ORDER BY cnt DESC (defn build-mini-profile [user-info] (let [st (fetch-template-fragment "mini_profile") nick (user-info :nick) - score (lookup-score nick)] + score (if use-redis-favscore + (fetch-redis-favscore nick) + (lookup-score nick))] (doseq [a [:nick :avatar :contact :bio]] (let [v (user-info a)] (.setAttribute st (name a) @@ -516,6 +522,7 @@ ORDER BY cnt DESC raw-dumps (if use-popular-redis (fetch-popular-dumps-redis profile-nick (:nick session)) (fetch-popular-dumps profile-nick (:nick session))) + raw-dumps (filter #(> (:count %) 0) raw-dumps) dumps (map process-message-for-output raw-dumps)] (.setAttribute st "nick" profile-nick) (.setAttribute st "mini_profile" (build-mini-profile user-info)) @@ -528,10 +535,8 @@ ORDER BY cnt DESC (def *per-directory-page* 25) (defn process-directory-entry [entry] - (let [score (lookup-score (:nick entry))] - (assoc (stringify-and-escape entry) - "score_ent" (score-to-entity score) - "score" score))) + (assoc (stringify-and-escape entry) + "score_ent" (score-to-entity (:score entry)))) (def directory-cache-ttl (minutes 10)) @@ -541,15 +546,25 @@ ORDER BY cnt DESC (defn add-recent-posts [user-id users] (if-not (empty? users) - (let [f (if user-id lookup-recent-posts lookup-recent-posts-tagless) - res (f user-id (map :user_id users))] + (let [res (if user-id + (lookup-recent-posts user-id (map :user_id users)) + (lookup-recent-posts-tagless (map :user_id users)))] (for [u users] (merge u (find-first #(= (:user_id u) (:user_id %)) res)))))) +(defn add-recent-posts-nick [user-id users] + (if-not (empty? users) + (let [nicks (map :nick users) + res (if user-id + (lookup-recent-posts-by-nicks user-id nicks) + (lookup-recent-posts-tagless-by-nicks nicks))] + (for [u users] + (merge u (find-first #(= (:nick u) (:nick %)) res)))))) + (defn get-directory-info [user-id offset] - (map process-directory-entry - (add-recent-posts user-id - (get-user-ranking offset *per-directory-page*)))) + (let [res (fetch-redis-directory offset *per-directory-page*)] + (map process-directory-entry + (add-recent-posts-nick user-id res)))) (defn directory [session offset] (let [st (fetch-template "directory" session) @@ -805,28 +820,31 @@ ORDER BY cnt DESC (try (do-insert "tags" ["user_id" "message_id" "tag"] - [(:user_id user) (msg :message_id) tag]) - (if (and (= tag "favorite") - (not (= (msg :nick) (user :nick)))) + [(:user_id user) (:message_id msg) tag]) + (when (and (= tag "favorite") + (not (= (msg :nick) (:nick user)))) + (if-not (:admin_only msg) + (incrby-redis-favscore! (:nick msg) (:message_id msg) 1)) (insert-fav-notification! (msg :nick) (user :nick) (user :avatar) (msg :content))) true ; catch error when inserting duplicate tags - (catch Exception e false))) + (catch Exception e + (do (println e) + false)))) (defn validated-add-tag [session params] (if (session :nick) (let [nick (session :nick) user-id (session :user_id) - user-admin? (session :admin-only) - msg-id (params :message_id) + msg-id (params :message_id) tag (validate-tag (params :tag)) msg (fetch-message-by-id msg-id) access (or (is-vip? session) - (not (:admin-only msg)))] + (not (:admin_only msg)))] (cond (not msg) (resp-error "NO_MSG") (not access) (resp-error "NO_MSG") (not tag) (resp-error "NO_TAG") @@ -835,10 +853,18 @@ ORDER BY cnt DESC (resp-error "TAG_EXISTS_ALREADY_OR_SOMETHING_ELSE_IS_FUCKED")))) (resp-error "NO_USER"))) -(defn remove-tag [user-id message-id tag] - (let [query "user_id = ? AND message_id = ? AND lower(tag) = ?"] - (do-delete "tags" [query user-id (maybe-parse-int message-id) (normalize-tag-for-db (.toLowerCase tag))]) - (resp-success "OK"))) +(defn remove-tag [user-id msg-id tag] + (let [query "user_id = ? AND message_id = ? AND lower(tag) = ?" + msg-id (maybe-parse-int msg-id) + tag (normalize-tag-for-db tag) + msg (fetch-message-by-id msg-id)] + (let [rows-deleted (first (do-delete "tags" [query user-id msg-id tag]))] + (if-not (zero? rows-deleted) + (do + (if-not (:admin_only msg) + (incrby-redis-favscore! (:nick msg) msg-id -1)) + (resp-success "OK")) + (resp-error "NO_TAG"))))) (defn validated-remove-tag [session params] (if (session :nick) @@ -1149,9 +1175,8 @@ ORDER BY cnt DESC (unknown-page))) (defn hall-of-fame [session] - (let [st (fetch-template "fame" session) - msgs (add-user-favs-to-msgs (poll hall-results) - (session :user_id))] + (let [st (fetch-template "fame" session) + msgs (fetch-redis-hall (:nick session))] (.setAttribute st "dumps" (map process-message-for-output msgs)) (.toString st))) @@ -1394,23 +1419,12 @@ ORDER BY cnt DESC (load-rooms!) (start! reserved-nicks) -(def server (start-server (options :port))) -(start! *active-mutes*) -; Delay the following to reduce start-load -(Thread/sleep 15000) -(start! *user-scores*) +(def server (start-server (options :port))) +(start! *active-mutes*) (start-user-flusher!) (start-session-pruner!) -(start! hall-results) - -;; Scott 2010/8/30: disable feeds to test impact on server load -;; (and see if anyone notices) -;; (if (= *server-url* "http://dump.fm") -;; (do (start! feed-downloader) -;; (start! feed-inserter))) - ;(if (not= *server-url* "http://dump.fm") ; (start! random-poster)) diff --git a/src/tags.clj b/src/tags.clj index 37c93b0..bc022f9 100644 --- a/src/tags.clj +++ b/src/tags.clj @@ -15,7 +15,8 @@ (.toLowerCase tag))) ; save all spaces in tags as dashes? -(defn normalize-tag-for-db [tag] (str tag)) +(defn normalize-tag-for-db [tag] + (lower-case tag)) ; (.replace tag " " "-")) ; todo: remove unicode escape sequences and line breaks and stuff? -- cgit v1.2.3-70-g09d2 From dd46cb29fa939546908db15fc92491bc49f3130f Mon Sep 17 00:00:00 2001 From: Scott Ostler Date: Mon, 29 Nov 2010 01:15:49 -0500 Subject: Commit initial vip-only direct messaging --- db/0-create.psql | 10 ++-- src/config.clj | 3 +- src/datalayer.clj | 70 +++++++++++++++++++++- src/feed.clj | 2 + src/message.clj | 39 ++++++++++++ src/rooms.clj | 13 +--- src/site.clj | 94 +++++++++++++++-------------- src/user.clj | 60 +++++++++++++++++-- src/utils.clj | 23 ------- static/js/pichat.js | 158 ++++++++++++++++++++++++++++--------------------- template/head.st | 4 +- template/profile.st | 20 ++++++- template/rooms/VIP.st | 3 + template/rooms/chat.st | 3 + 14 files changed, 338 insertions(+), 164 deletions(-) create mode 100644 src/message.clj (limited to 'src/config.clj') diff --git a/db/0-create.psql b/db/0-create.psql index 4fa8536..762bc3a 100644 --- a/db/0-create.psql +++ b/db/0-create.psql @@ -137,11 +137,11 @@ CREATE TABLE invalid_feed_images ( CREATE INDEX invalid_feed_images_idx ON invalid_feed_images (image_url); -CREATE TABLE events ( - event_id SERIAL PRIMARY KEY, - name text NOT NULL, - author integer NOT NULL REFERENCES, - created_on timestamp NOT NULL DEFAULT now() +CREATE TABLE direct_messages ( + dm_id SERIAL PRIMARY KEY, + message_id integer NOT NULL REFERENCES messages, + author_id integer NOT NULL REFERENCES users, + recip_id integer NOT NULL REFERENCES users ); -- dont add this yet diff --git a/src/config.clj b/src/config.clj index 5253379..c4e2fe3 100644 --- a/src/config.clj +++ b/src/config.clj @@ -31,4 +31,5 @@ (def *dumps-per-page* 20) (def *vip-dumps-per-page* 200) (def message-count-limit 200) -(def num-hall-dumps 50) \ No newline at end of file +(def num-hall-dumps 50) +(def max-content-size 2468) \ No newline at end of file diff --git a/src/datalayer.clj b/src/datalayer.clj index 139274b..28ef3bf 100644 --- a/src/datalayer.clj +++ b/src/datalayer.clj @@ -1,8 +1,13 @@ (ns datalayer (:require redis tags) - (:use config + (:use clojure.contrib.sql + clojure.contrib.json.write + clojure.contrib.json.read + config jedis + message + user utils)) @@ -74,8 +79,6 @@ WHERE u.nick = ANY(?)" AND m.message_id = ?"] (first (do-select [query (maybe-parse-int m-id -1)])))) - - ;;;; Popular Posts (def popular-dumps-qry " @@ -145,3 +148,64 @@ order by count desc limit ? offset ?") (redis/zrevrange "hall" 0 (dec num-hall-dumps))))] (if-not (empty? ids) (tags/fetch-dumps-by-ids ids viewer-nick)))) + +;;;; Message insertion + +(def msg-insert-query + "INSERT INTO messages (user_id, room_id, content, is_image, is_text) + VALUES (?, ?, ?, ?, ?) RETURNING message_id, created_on") + +(defn insert-message-into-postgres! [author-id room-id content is-image is-text recips] + (with-connection *db* + (transaction + (let [{msg-id :message_id ts :created_on} + (first + (do-select [msg-insert-query + author-id room-id content is-image is-text]))] + (doseq [r recips] + (insert-values + :direct_messages + [:message_id :author_id :recip_id] + [msg-id author-id (:user_id r)])) + [msg-id ts])))) + +(defn insert-recips-into-redis! [recips author-id ts content] + (let [dm-json (json-str {"author_id" author-id + "recips" (map :nick recips) + "content" content})] + (redis/with-server redis-server + (redis/atomically + (doseq [r recips] + (redis/zadd (str "directmessage:" (:user_id r)) + (.getTime ts) + dm-json)))))) + +(defn insert-message! [author-id author-nick room-id content] + (let [msg-type (classify-msg content) + is-image (boolean (#{:image :mixed} msg-type)) + is-text (boolean (#{:mixed :text} msg-type)) + recips (get-recips content) + [msg-id ts] (insert-message-into-postgres! author-id + room-id + content + is-image + is-text + recips)] + (if-not (empty? recips) + (insert-recips-into-redis! recips author-id ts content)) + {:author author-nick + :msg-id msg-id + :room room-id + :db-ts ts + :content content + :recips (map (comp lower-case :nick) recips)})) + +(defn fetch-private-messages [user-id] + (for [dm (redis/with-server redis-server + (redis/zrevrange (str "directmessage:" user-id) 0 40))] + (let [dm (read-json dm) + info (fetch-user-id (get dm "author_id"))] + {"nick" (:nick info) + "content" (get dm "content") + "recips" (get dm "recips") + "avatar" (:avatar info)}))) diff --git a/src/feed.clj b/src/feed.clj index c8454d0..898f085 100755 --- a/src/feed.clj +++ b/src/feed.clj @@ -16,6 +16,8 @@ scheduled-agent utils)) +;; DEPRECATED + (def *feeds-path* "docs/feeds.csv") (defn queue-image! [room-key img] diff --git a/src/message.clj b/src/message.clj new file mode 100644 index 0000000..a8e0e9b --- /dev/null +++ b/src/message.clj @@ -0,0 +1,39 @@ +(ns message + (:use user)) + +;; Message parsing + +;; http://snippets.dzone.com/posts/show/6995 +(def url-regex #"(?i)^((http\:\/\/|https\:\/\/|ftp\:\/\/)|(www\.))+(\w+:{0,1}\w*@)?(\S+)(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?$") +(def pic-regex #"(?i)\.(jpg|jpeg|png|gif|bmp|svg)(\?|&|$)") + +(defn is-image? [word] + (and (re-find url-regex word) + (re-find pic-regex word))) + +(defn take-images [content] + (filter is-image? (.split content " "))) + +(defn classify-msg [msg] + (let [words (.split msg " ") + imgs (map is-image? words)] + (cond (every? boolean imgs) :image + (some boolean imgs) :mixed + :else :text))) + +(def recip-regex #"(?:^|\s)@\w+") + +(defn get-recips [content] + (filter + boolean + (for [at-nick (re-seq recip-regex content)] + (fetch-nick (.substring (.trim at-nick) 1))))) + +(defn get-recips-from-msgs [msgs] + (let [recips (set (apply concat + (for [m msgs] + (re-seq recip-regex (:content m)))))] + (filter + boolean + (for [r recips] + (fetch-nick (.substring (.trim r) 1)))))) diff --git a/src/rooms.clj b/src/rooms.clj index 574532a..3367ea9 100644 --- a/src/rooms.clj +++ b/src/rooms.clj @@ -6,7 +6,7 @@ utils user)) -(defstruct message-struct :nick :content :created_on :msg_id) +(defstruct message-struct :nick :content :created_on :msg_id :recips) (def *run-flusher* true) (def *flusher-sleep* (seconds 4)) @@ -99,19 +99,12 @@ ; Note: To ensure that the msg's timestamp is consistent ; with other msg creations, build-msg must be used ; within a dosync. -(defn build-msg [nick content msg-id] - (struct message-struct nick content (new Date) msg-id)) +(defn build-msg [nick content msg-id recips] + (struct message-struct nick content (new Date) msg-id recips)) (defn add-message [msg room] (insert-and-truncate! (room :messages) msg message-count-limit)) -(defn insert-message-into-db! [user-id room-id content is-image] - (:message_id - (first - (do-select ["INSERT INTO messages (user_id, room_id, content, is_image) - VALUES (?, ?, ?, ?) RETURNING message_id" - user-id room-id content is-image])))) - (defn create-and-add-room! [key] (do-select ["INSERT INTO rooms (key, name, description) VALUES (?, ?, ?) RETURNING room_id" diff --git a/src/site.clj b/src/site.clj index cf6ae14..7db43e7 100644 --- a/src/site.clj +++ b/src/site.clj @@ -21,10 +21,10 @@ datalayer email fame + message utils cookie-login session-sweeper - feed rooms tags scheduled-agent @@ -119,14 +119,13 @@ ;; User-id/nick cache ;; I keep needing to grab user-id from a nick so I thought I'd cache them -;; @timb: I just duplicated this in the user-info map :( -;; we should reconcile our user caches -(def user-id-cache (ref {})) +;; sostler todo: will replace this w/ user/user-id-cache soon +(def *user-id-cache* (ref {})) (def *user-id-cache-size* 500) (defn user-id-from-nick [nick] (let [nick (lower-case nick) - found (@user-id-cache nick)] + found (@*user-id-cache* nick)] (if found found (let [query (str "SELECT user_id FROM users WHERE lower(nick) = ?") @@ -135,8 +134,8 @@ nil (let [found (res :user_id)] (dosync - (if (> (count @user-id-cache) *user-id-cache-size*) (ref-set user-id-cache {})) - (alter user-id-cache assoc nick found)) + (if (> (count @*user-id-cache*) *user-id-cache-size*) (ref-set *user-id-cache* {})) + (alter *user-id-cache* assoc nick found)) found)))))) ;; Login code @@ -384,7 +383,7 @@ ORDER BY cnt DESC (if-let [user-info (fetch-nick profile-nick)] (let [st (fetch-template template session) profile-nick (:nick user-info) ; Update to get right casing - nick (session :nick) + nick (:nick session) logger (make-time-logger) is-home (and nick (= nick profile-nick)) score (if use-redis-favscore @@ -394,6 +393,10 @@ ORDER BY cnt DESC :user-tag-id (:user_id session) :nick profile-nick :limit 10) + dms (if-vip + (fetch-private-messages (:user_id user-info))) + recips (if dms + (set (concat (map #(get % "recips") dms)))) imgs (pull-random-dump-images dumps 5)] (do (.setAttribute st "is_home" is-home) @@ -403,6 +406,9 @@ ORDER BY cnt DESC (if (non-empty-string? v) (escape-html v))))) (.setAttribute st "score" (comma-format score)) (.setAttribute st "score_ent" (score-to-entity score)) + (when-not (empty? dms) + (.setAttribute st "dms" dms) + (.setAttribute st "recips" (json-str (map lower-case recips)))) (if (not (empty? imgs)) (.setAttribute st "imgs" imgs)) (.setAttribute st "debug_log_items" (logger)) @@ -411,7 +417,8 @@ ORDER BY cnt DESC (defn update-user-db [user-id attr val] (with-connection *db* - (update-values "users" ["user_id = ?" user-id] {attr val}))) + (update-values "users" ["user_id = ?" user-id] {attr val})) + (update-cache! user-id attr val)) (defn update-avatar [session url] (update-user-db (session :user_id) "avatar" url) @@ -614,6 +621,7 @@ ORDER BY cnt DESC :user-tag-id (:user_id session) :hide-vip false :limit (:history_size room))) + recips (map :nick (get-recips-from-msgs raw-msgs)) message-list (to-array (map process-message-for-output raw-msgs))] (if nick (dosync @@ -621,6 +629,7 @@ ORDER BY cnt DESC (doto st (.setAttribute "users" (prepare-user-list room true)) (.setAttribute "messages" message-list) + (.setAttribute "recips" (json-str (map lower-case recips))) (.setAttribute "roomkey" (room :key)) (.setAttribute "isadminroom" (room :admin_only)) (.setAttribute "json_room_key" (json-str (room :key))) @@ -670,34 +679,29 @@ ORDER BY cnt DESC (str "" content "")) (str content))) -(defn msg-db [user-id room-id content] - (let [msg-type (classify-msg content) - is-image (boolean (#{:image :mixed} msg-type)) - is-text (boolean (#{:mixed :text} msg-type)) - qry (str "INSERT INTO messages (user_id, room_id, content, is_image, is_text) " - "VALUES (?, ?, ?, ?, ?) RETURNING message_id")] - (with-connection *db* - ((first (do-select [qry user-id room-id content is-image is-text])) - :message_id)))) - (defn msg [session params] - (let [user-id (session :user_id) - mute (get (poll *active-mutes*) user-id) - nick (session :nick) - room-key (params :room) - room (lookup-room room-key) - content (.trim (params :content))] - (cond (not room) (resp-error "BAD_ROOM") - (not nick) (resp-error "NOT_LOGGED_IN") - mute (resp-error (format-mute mute)) + (let [user-id (session :user_id) + mute (get (poll *active-mutes*) user-id) + nick (session :nick) + room-key (params :room) + room (lookup-room room-key) + content (.trim (params :content)) + content-too-long? (> (count content) + max-content-size)] + (cond (not room) (resp-error "BAD_ROOM") + (not nick) (resp-error "NOT_LOGGED_IN") + content-too-long? (resp-error "TOO_LONG") + mute (resp-error (format-mute mute)) :else - (let [content (validated-content content session) - msg-id (msg-db user-id (room :room_id) content)] + (let [content (validated-content content session) + msg-info (insert-message! user-id nick (:room_id room) content) + msg-id (:msg-id msg-info)] (dosync (if (not (contains? (ensure (room :users)) nick)) (login-user (user-struct-from-session session) room)) - (add-message (build-msg nick content msg-id) room)) - (resp-success msg-id))))) + (add-message (build-msg nick content msg-id (:recips msg-info)) room)) + (resp-success {:msgid msg-id + :recips (:recips msg-info)}))))) (defn validated-msg [session params request] @@ -737,7 +741,8 @@ ORDER BY cnt DESC dump-offset (* offset *dumps-per-page*) image-only (and (not (room :admin_only)) (not= (params :show) "all")) - raw-dumps (logger tags/fetch-dumps-by-room :room-id (room :room_id) + raw-dumps (logger tags/fetch-dumps-by-room + :room-id (room :room_id) :image-only image-only :amount (+ 1 *dumps-per-page*) :offset dump-offset) @@ -1113,21 +1118,20 @@ ORDER BY cnt DESC ; errors. ; The upload code doesn't use jQuery.ajax, and doesn't JSON-eval ; responses. Therefore, return strings should not be JSON-encoded. - (defn do-upload [session image room] (if-let [err (validate-upload-file (image :tempfile) room)] (resp-error err) - (let [filename (format-filename (:filename image) (session :nick)) - date (today) - dest (open-file [*image-directory* date] filename) - url (image-url-from-file "images" date dest) - msg-id (msg-db (session :user_id) (room :room_id) url) - msg (struct message-struct (session :nick) url (new Date) msg-id)] - (do - (dosync - (add-message msg room)) - (copy (:tempfile image) dest) - [200 "OK"])))) + (let [filename (format-filename (:filename image) (session :nick)) + date (today) + dest (open-file [*image-directory* date] filename) + url (image-url-from-file "images" date dest) + msg-info (insert-message! (:user_id session) (:nick session) + (:room_id room) url)] + (copy (:tempfile image) dest) + (dosync + (let [msg (build-msg (:nick session) url (:msg-id msg-info) (:recips msg-info))] + (add-message msg room))) + [200 "OK"]))) (defn upload [session params request] (let [room-key (params :room) diff --git a/src/user.clj b/src/user.clj index 1d59944..7641bd8 100644 --- a/src/user.clj +++ b/src/user.clj @@ -16,12 +16,62 @@ (> (count n) 16) "NICK_TOO_LONG" (not (re-matches *nick-regex* n)) "NICK_INVALID_CHARS")) +;;; User info cache + +(def user-cache-size 500) +(def user-nick-cache (ref {})) +(def user-id-cache (ref {})) + +(defn update-cache! [uid attr val] + (dosync + (if-let [info (get @user-id-cache uid)] + (let [nick (lower-case (:nick info)) + new-info (assoc info attr val)] + (alter user-id-cache assoc uid new-info) + (alter user-nick-cache assoc nick new-info))))) + + (defn fetch-nick [nick] - (let [q1 "SELECT * FROM users WHERE nick = ? LIMIT 1" - ; ORDER BY ensures consistent retrieval of ambiguious names - q2 "SELECT * FROM users WHERE lower(nick) = ? ORDER BY nick LIMIT 1"] - (or (first-or-nil (do-select [q1 nick])) - (first-or-nil (do-select [q2 (lower-case nick)]))))) + (let [lcnick (lower-case nick)] + (if (contains? user-nick-cache lcnick) + (get user-nick-cache lcnick) + (let [info (first + (do-select ["SELECT * FROM users WHERE lower(nick) = ? LIMIT 1" + lcnick])) + user-id (:user_id info)] + (dosync + (alter user-nick-cache assoc lcnick info) + (if (and info user-id) + (alter user-id-cache assoc user-id info))) + info)))) + +(defn fetch-nicks [nicks] + (let [lcnicks (map lower-case nicks) + cache @user-nick-cache + to-fetch (filter #(not (contains? cache %)) lcnicks) + fetched-info (do-select ["SELECT * FROM users WHERE lower(nick) = ANY(?)" + (sql-array "text" to-fetch)]) + info-map (zipmap (map (comp lower-case :nick) fetched-info) + fetched-info)] + (doseq [nick to-fetch] + (let [info (get info-map nick)] + (dosync + (alter user-nick-cache assoc nick info) + (if info + (alter user-id-cache assoc (:user_id info) info))))) + (filter + boolean + (for [nick lcnicks] + (get @user-nick-cache nick))))) + +(defn fetch-user-id [uid] + (if (contains? @user-id-cache uid) + (get @user-id-cache uid) + (if-let [info (first + (do-select ["SELECT * FROM users WHERE user_id = ? LIMIT 1" uid]))] + (dosync + (alter user-nick-cache assoc (lower-case (:nick info)) info) + (alter user-id-cache assoc uid info))))) (defn authorize-nick-hash [nick hash] (let [db-user (fetch-nick nick)] diff --git a/src/utils.clj b/src/utils.clj index 84454cd..8aaffba 100755 --- a/src/utils.clj +++ b/src/utils.clj @@ -36,26 +36,6 @@ (.setMaxConnections 10))})) -;; Message parsing - -;; http://snippets.dzone.com/posts/show/6995 -(def url-regex #"(?i)^((http\:\/\/|https\:\/\/|ftp\:\/\/)|(www\.))+(\w+:{0,1}\w*@)?(\S+)(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?$") -(def pic-regex #"(?i)\.(jpg|jpeg|png|gif|bmp|svg)(\?|&|$)") - -(defn is-image? [word] - (and (re-find url-regex word) - (re-find pic-regex word))) - -(defn take-images [content] - (filter is-image? (.split content " "))) - -(defn classify-msg [msg] - (let [words (.split msg " ") - imgs (map is-image? words)] - (cond (every? boolean imgs) :image - (some boolean imgs) :mixed - :else :text))) - ;; Misc (defn except! [& more] @@ -415,9 +395,6 @@ (defn serve-template [template session] (.toString (fetch-template template session))) -(defn first-or-nil [l] - (if (empty? l) nil (first l))) - ;; VIP (defn is-vip? [session] diff --git a/static/js/pichat.js b/static/js/pichat.js index 6a6d962..0a849eb 100644 --- a/static/js/pichat.js +++ b/static/js/pichat.js @@ -119,6 +119,8 @@ Log.initialize(); URLRegex = /((\b(http\:\/\/|https\:\/\/|ftp\:\/\/)|(www\.))+(\w+:{0,1}\w*@)?(\S+)(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?)/gi; PicRegex = /\.(jpg|jpeg|png|gif|bmp|svg|fid)$/i; +RecipRegex = /(^|\s)@\w+/g; + function getImagesAsArray(text) { var imgs = [] @@ -133,10 +135,24 @@ function getImagesAsArray(text) { return imgs } -function linkify(text) { - LastMsgContainsImage = false - text = text.replace(URLRegex, linkReplace); - return text +function linkify(text, recips) { + LastMsgContainsImage = false; + var recipWrapper = function(text) { return recipientReplace(text, recips); }; + return text.replace(URLRegex, linkReplace).replace(RecipRegex, recipWrapper); +} + +function recipientReplace(atText, recips) { + if (atText[0] == ' ') { + atText = atText.slice(1); + var space = ' '; + } else { + var space = ''; + } + var nick = atText.slice(1); + if (!recips || recips.indexOf(nick.toLowerCase()) == -1) { + return space + atText; + } else + return space + '' + atText + ''; } // use this in escapeHtml to turn everyone's text lIkE tHiS @@ -312,22 +328,23 @@ function setImgsEnable() { } }; -function buildMsgContent(content) { +function buildMsgContent(content, recips) { if (content.substr(0,6) == "") return content.substr(6,content.length - 13) - else return linkify(escapeHtml(content)); + else return linkify(escapeHtml(content), recips); } // todo: // isLoading doesn't get passed the right thing by $.map in addMessages -function buildMessageDiv(msg, isLoading) { +function buildMessageDiv(msg, opts) { + var opts = opts || {}; var nick = escapeHtml(msg.nick); removeOldMessages(); - var builtContent = buildMsgContent(msg.content); + var builtContent = buildMsgContent(msg.content, opts.recips); var msgId = ('msg_id' in msg) ? 'id="message-' + msg.msg_id + '"' : ''; - var loadingClass = isLoading ? ' loading' : ''; + var loadingClass = opts.isLoading ? ' loading' : ''; var containsImageClass = LastMsgContainsImage ? ' contains-image' : ''; var displayStyle = ((ImgsEnabled && LastMsgContainsImage) || (TextEnabled && !LastMsgContainsImage)) ? '' : ' style="display: none"'; @@ -436,55 +453,56 @@ function clearMessages(){ } function submitMessage() { - var content = $.trim($('#msgInput').val()); - - if (content == "/clear") { - clearMessages() + var content = $.trim($('#msgInput').val()); + + if (content == "/clear") { + clearMessages() + $('#msgInput').val(''); + return; + } + + var invalidDomain = invalidImageDomain(content); + if (invalidDomain) { + $('#msgInput').blur(); // Remove focus to prevent FF alert loop + alert("Sorry, cannot accept images from " + invalidDomain + ". Maybe host the image elsewhere?"); + return; + } + $('#msgInput').val(''); - return; - } - - var invalidDomain = invalidImageDomain(content); - if (invalidDomain) { - $('#msgInput').blur(); // Remove focus to prevent FF alert loop - alert("Sorry, cannot accept images from " + invalidDomain + ". Maybe host the image elsewhere?"); - return; - } - - $('#msgInput').val(''); - if (content == '') { return; } - if (content.length > 2468) { - alert("POST TOO LONG DUDE!"); - return; - } // this shouldn't just be client side :V - PendingMessages[content] = true; - - var msg = { 'nick': Nick, 'content': content }; - var div = addNewMessage(msg, true); - - var onSuccess = function(json) { - if (typeof pageTracker !== 'undefined') { - pageTracker._trackEvent('Message', 'Submit', - typeof Room !== 'undefined' ? Room : 'UnknownRoom'); + if (content == '') { return; } + if (content.length > 2468) { + alert("POST TOO LONG DUDE!"); + return; } - div.attr('id', 'message-' + json) - .removeClass('loading').addClass('loaded'); - }; - var onError = function(resp, textStatus, errorThrown) { - div.remove(); - handleMsgError(resp); - }; - - $.ajax({ - type: 'POST', - timeout: 15000, - url: '/msg', - data: { 'room': Room, 'content': content }, - cache: false, - dataType: 'json', - success: onSuccess, - error: onError - }); + PendingMessages[content] = true; + + var msg = { 'nick': Nick, 'content': content }; + var div = addNewMessage(msg, true); + + var onSuccess = function(json) { + if (typeof pageTracker !== 'undefined') { + pageTracker._trackEvent('Message', 'Submit', + typeof Room !== 'undefined' ? Room : 'UnknownRoom'); + } + div.attr('id', 'message-' + json.msgid) + .removeClass('loading').addClass('loaded'); + div.find('.content').html(buildMsgContent(content, json.recips)); + }; + var onError = function(resp, textStatus, errorThrown) { + div.remove(); + handleMsgError(resp); + }; + + $.ajax({ + type: 'POST', + timeout: 15000, + url: '/msg', + data: { 'room': Room, 'content': content }, + cache: false, + dataType: 'json', + success: onSuccess, + error: onError + }); } function ifEnter(fn) { @@ -493,13 +511,14 @@ function ifEnter(fn) { }; } -function addNewMessages(msgs) { - var msgStr = $.map(msgs, buildMessageDiv).join(''); +function addNewMessages(msgs, recips) { + var msgOpts = { recips: recips }; + var msgStr = $.map(msgs, function(msg) { buildMessageDiv(msg, msgOpts).join(''); }); $('#messageList').append(msgStr); } function addNewMessage(msg, isLoading) { - var msgStr = buildMessageDiv(msg, isLoading); + var msgStr = buildMessageDiv(msg, { isLoading: true }); var div = $(msgStr).appendTo('#messageList'); return div; } @@ -516,11 +535,11 @@ function flattenUserJson(users) { return s; } -function updateUI(msgs, users, favs) { +function updateUI(msgs, users, favs, recips) { if (window['growlize'] && msgs && msgs.length > 0) { $.map(msgs, buildGrowlDataAndPopDatShit) } else if (msgs && msgs.length > 0) { - addNewMessages(msgs); + addNewMessages(msgs, recips); } if (users !== null) { var flattened = flattenUserJson(users); @@ -552,14 +571,13 @@ function isDuplicateMessage(m) { function refresh() { var onSuccess = function(json) { try { - Timestamp = json.timestamp; - + Timestamp = json.timestamp; $.map(json.messages, function(msg){ MessageContentCache[msg.msg_id.toString()] = msg.content }) var messages = $.grep( json.messages, function(m) { return !isDuplicateMessage(m) }); - updateUI(messages, json.users, json.favs); + updateUI(messages, json.users, json.favs, json.recips); if (!Away.HasFocus) Away.UnseenMsgCounter += messages.length; } catch(e) { @@ -619,7 +637,7 @@ function initChat() { var dump = $(this); var content = dump.find(".content") MessageContentCache[dump.attr("id").substr(8)] = content.text() - content.html(buildMsgContent(content.text())); + content.html(buildMsgContent(content.text(), Recips)); if ((ImgsEnabled && dump.hasClass('contains-image')) || (TextEnabled && !dump.hasClass('contains-image'))) dump.show(); @@ -698,11 +716,15 @@ function enableProfileEdit() { } function initProfile() { - Search.initInpage() - $(".linkify").each(function() { + Search.initInpage(); + $(".linkify-text").each(function() { var text = jQuery(this).text(); jQuery(this).html(linkifyWithoutImage(text)); - }); + }); + + $(".linkify-full").each(function() { + $(this).html(buildMsgContent($(this).text(), Recips)); + }); $('#edit-toggle').click(enableProfileEdit); activateProfileEditable(); diff --git a/template/head.st b/template/head.st index 66be7c1..a3ff733 100644 --- a/template/head.st +++ b/template/head.st @@ -1,10 +1,10 @@ - + - + diff --git a/template/profile.st b/template/profile.st index f66069e..6f44058 100644 --- a/template/profile.st +++ b/template/profile.st @@ -50,14 +50,14 @@

contact info

$if(contact)$ -
$contact$
+
$contact$
$else$ $endif$

bio

$if(bio)$ -
$bio$
+
$bio$
$else$ $endif$ @@ -90,7 +90,23 @@ style="border:none; width:500px; height:30px"> + + $if(dms)$ +
+
+

Messages

+ $dms: { dm | +
+ $dm.nick$ + + $dm.content$
+ + }$ +
+ $endif$
diff --git a/template/rooms/VIP.st b/template/rooms/VIP.st index f078d0d..fa38eb5 100644 --- a/template/rooms/VIP.st +++ b/template/rooms/VIP.st @@ -110,6 +110,9 @@
$messagepane()$ +
diff --git a/template/rooms/chat.st b/template/rooms/chat.st index b96456a..f7ca888 100644 --- a/template/rooms/chat.st +++ b/template/rooms/chat.st @@ -110,6 +110,9 @@
$messagepane()$ +
-- cgit v1.2.3-70-g09d2