timb: image altars

4 files changed, 182 insertions, 3 deletions

diff --git a/db/0-create.psql b/db/0-create.psql
index 1162f86..d2d4bfc 100644
--- a/db/0-create.psql
+++ b/db/0-create.psql
@@ -57,7 +57,12 @@ CREATE INDEX messages_user_created_on_image_only_idx ON messages (user_id, creat
 -- historical image dumps in a room. needs to handle non-image messages.
 CREATE INDEX messages_room_id_created_on_idx ON messages (room_id, created_on);
 
-
+CREATE TABLE image_altars (
+  message_id integer NOT NULL,
+  user_id integer NOT NULL,
+  PRIMARY KEY(message_id)
+);
+  
 CREATE TABLE image_urls (
   url text,
   last_posted timestamp NOT NULL DEFAULT now(),
@@ -150,4 +155,4 @@ CREATE TABLE avatars (
 CREATE TABLE images (
   image_id SERIAL PRIMARY KEY,
   
-);
\ No newline at end of file
+);
diff --git a/scripts/fill.image_altars.py b/scripts/fill.image_altars.py
new file mode 100644
index 0000000..4b8dd91
--- /dev/null
+++ b/scripts/fill.image_altars.py
@@ -0,0 +1,105 @@
+# this needs python 3
+
+import re
+import sys
+import postgresql
+from urllib.parse import urlparse
+
+db = postgresql.open("pq://postgres:root@localhost/dumpfm")
+db.execute("SET CLIENT_ENCODING to 'UNICODE'")
+
+def get_highest_message_id_in_db():
+  ps = db.prepare("SELECT message_id FROM image_altars ORDER BY message_id DESC LIMIT 1")
+  try:
+    highest = int(ps()[0][0])
+  except IndexError:
+    highest = 0
+  return highest
+
+def add_altar(message_id, user_id, content):
+  try:
+    print(message_id, content)
+  except UnicodeEncodeError:
+    print("i thought python 3 fixed the unicode shit. yet i still get unicode errors everywhere. GOOD JOB FUCKHEADS")
+  ps = db.prepare("INSERT INTO image_altars(message_id, user_id) VALUES($1, $2)")
+  try:
+    ps(message_id, user_id)
+  except postgresql.exceptions.UniqueError:
+    print("skipped adding a dupe")
+
+# NOTE. hardcoded room numbers to index here... only indexing DUMPFM (1) and GIF (8) currently.
+def get_messages(lower, upper):
+  ps = db.prepare("SELECT message_id, user_id, content FROM messages WHERE message_id >= $1 AND message_id <= $2 AND room_id IN (1,8) ORDER BY message_id ASC")
+  rows = ps(lower, upper)
+  return rows
+
+def is_url_an_image(url):
+  image_types = {"jpg", "bmp", "gif", "png"}
+  url = urlparse(url)
+  filetype = url.path[-3:].lower()
+  return filetype in image_types
+
+def is_altar(content):
+  if content[0:6] == "<safe>": # skip html messages
+    return False
+  tokens = content.split(" ")
+  if is_bad_sized_array(tokens): # no even sized arrays
+#    print("array not oddly sized")
+    return False
+  for token in tokens: # everything must be an image
+    if token[0:7] != "http://":
+#      print("contains stuff thats not urls")
+      return False
+    elif not is_url_an_image(token):
+#      print("contains stuff thats not images")
+      return False
+  middleImage = tokens[int((len(tokens)-1)/2)]
+  i = 0
+  while i < (len(tokens)-1)/2:
+    if tokens[i] != tokens[len(tokens) - 1 - i]: # must be symmetric
+#      print("not symmetric")
+      return False
+    if tokens[i] == middleImage: # middle image must be unique
+#      print("middle image not unique")
+      return False
+    i += 1
+  return True
+
+def process_messages(messages):
+  num_added = 0
+  processed = 0
+  for message in messages:
+    if processed % 1000 == 0:
+      print(processed, " processed so far")
+      processed += 1
+    if is_altar(message[2]):
+      add_altar(message[0], message[1], message[2])
+      num_added += 1
+  return num_added
+
+def get_urls_from_messages(messages):
+  urls = []
+  for message in messages:
+    urls.extend(get_images_from_messages(message[0]))
+  return urls
+
+# image altars look like aba or abcba but not a or abba
+def is_bad_sized_array(a):
+  if len(a) % 2 == 0:
+    return True
+  elif len(a) < 3:
+    return True
+  else:
+    return False
+
+if __name__ == "__main__":
+  if not len(sys.argv) == 2:
+    print('usage: fill.image_altars.py message_id_end')
+    sys.exit(1)
+
+  upper = int(sys.argv[1])
+
+  highest = get_highest_message_id_in_db()
+  messages = get_messages(highest, upper)
+  num_added = process_messages(messages)
+  print("added ", num_added, " altars to db")
diff --git a/src/site.clj b/src/site.clj
index f88afaf..620756d 100644
--- a/src/site.clj
+++ b/src/site.clj
@@ -847,6 +847,35 @@ WHERE  u.user_id = ANY(?)"
       (log session (lookup-room room-key) offset params)

       (resp-error "UNKNOWN_ROOM"))))

 

+;; Altars

+

+(defn altar-log [session params]

+  (let [id         (params :id)

+        st         (fetch-template "altar_log" session)

+        raw-dumps  (tags/fetch-altars :message-id id :amount (+ 1 *dumps-per-page*))

+        dumps      (map tags/add-favorited-flag (take *dumps-per-page* raw-dumps) (repeat session))

+        dumps      (map tags/remove-tags-for-output dumps)

+        dumps      (map process-message-for-output dumps)]

+    (.setAttribute st "dumps" dumps)

+    (if (> (count raw-dumps) *dumps-per-page*)

+      (.setAttribute st "next" ((last raw-dumps) :message_id)))

+    (.toString st)))

+

+(defn altar-log-by-nick [session params]

+  (let [id         (params :id)

+        nick       (params :nick)

+        user-id    (user-id-from-nick nick)

+        st         (fetch-template "altar_user_log" session)

+        raw-dumps  (tags/fetch-altars :message-id id :amount (+ 1 *dumps-per-page*) :user-id user-id)

+        dumps      (map tags/add-favorited-flag (take *dumps-per-page* raw-dumps) (repeat session))

+        dumps      (map tags/remove-tags-for-output dumps)

+        dumps      (map process-message-for-output dumps)]

+    (.setAttribute st "dumps" dumps)

+    (.setAttribute st "nick" nick)

+    (if (> (count raw-dumps) *dumps-per-page*)

+      (.setAttribute st "next" ((last raw-dumps) :message_id)))

+    (.toString st)))

+

 ;; Tags 

 

 (defn undecoded-url-piece [url position]

@@ -1263,6 +1292,10 @@ WHERE  u.user_id = ANY(?)"
   (POST "/cmd/tag/add" (validated-add-tag session params))

   (POST "/cmd/tag/rm" (validated-remove-tag session params))

 

+  (GET "/altars" (altar-log session params))

+  (GET "/altars/" (altar-log session params))

+  (GET "/altars/:id" (altar-log session params))

+

   ;; Events

   (GET "/event" (current-event session))

 

@@ -1313,6 +1346,9 @@ WHERE  u.user_id = ANY(?)"
   ;; Put username routes below all others in priority

   (GET "/:nick"           (profile session (params :nick)))

   (GET "/:nick/"          (profile session (params :nick)))

+  (GET "/:nick/altars"     (altar-log-by-nick session params))

+  (GET "/:nick/altars/"    (altar-log-by-nick session params))

+  (GET "/:nick/altars/:id" (altar-log-by-nick session params))

   (GET "/:nick/tag/:tag" (tagged-dumps-by-nick session params (request-url request)))

   (GET "/:nick/tag/:tag/:offset" (tagged-dumps-by-nick session params (request-url request)))

   (GET "/:nick/favorites" (favorites-handler session (params :nick) nil nil))

diff --git a/src/tags.clj b/src/tags.clj
index 3abeee9..6354e37 100644
--- a/src/tags.clj
+++ b/src/tags.clj
@@ -300,12 +300,45 @@ WHERE EXISTS
     AND r.room_id = m.room_id "
     (if include-vip "" "AND r.admin_only = false")))
 
+(defnk fetch-altars-query [:with-tags true :with-message-id false :with-user-id false] (str
+" SELECT
+    m.content, m.message_id, m.created_on,
+    u.nick, u.avatar, r.key "
+    (if with-tags ",
+      array_to_string(ARRAY(SELECT nick || ' ' || tag
+        FROM tags, users
+        WHERE message_id = m.message_id AND tags.user_id = users.user_id), ' ') as tags " "")
+" FROM (SELECT
+          message_id
+          FROM image_altars "
+          (if (and with-message-id (not with-user-id)) "WHERE message_id < ? " "") ;; todo: fix awful code
+          (if (and (not with-message-id) with-user-id) "WHERE user_id = ? " "")
+          (if (and with-message-id with-user-id) "WHERE message_id < ? AND user_id = ? " "")
+        " ORDER BY message_id DESC
+          LIMIT ? OFFSET ?
+  ) as sq, messages m, users u, rooms r
+  WHERE m.message_id = sq.message_id
+    AND m.user_id = u.user_id
+    AND r.room_id = m.room_id "
+))
+
 
 (defn fetch-dump-by-id [m-id]
   (let [query (fetch-dump-by-message-id-query)]
     (let [rows (do-select [query (maybe-parse-int m-id -1)])]
       (first (map parse-tags-from-row-as-tag-map rows)))))
 
+(defnk fetch-altars [:message-id 0 :user-id 0 :amount *dumps-per-page* :offset 0]
+  (let [message-id      (maybe-parse-int message-id 0)
+        with-message-id (> message-id 0)
+        with-user-id    (> user-id 0)
+        query           (fetch-altars-query :with-message-id with-message-id :with-user-id with-user-id)
+        query-vars      [amount offset]
+        query-vars      (if with-user-id (concat [user-id] query-vars) query-vars)
+        query-vars      (if with-message-id (concat [message-id] query-vars) query-vars)
+        rows            (do-select (vec (concat [query] query-vars)))]
+    (map parse-tags-from-row-as-tag-map rows)))
+
 (defnk fetch-dumps-by-room [:room-id 1 :image-only true :amount *dumps-per-page* :offset 0]
   (let [query (fetch-dumps-by-room-query image-only)]
     (let [rows (do-select [query room-id amount offset])]
@@ -365,4 +398,4 @@ WHERE EXISTS
                      :msg    msg
                      :added  (new Date)}
          new-fs     (conj cur-favs fav)]
-     (alter fav-map assoc to-nick new-fs))))
\ No newline at end of file
+     (alter fav-map assoc to-nick new-fs))))