add incremental daily hall building

1 files changed, 54 insertions, 8 deletions

diff --git a/src/redisload.clj b/src/redisload.clj
index 1ca7815..d05d9ed 100644
--- a/src/redisload.clj
+++ b/src/redisload.clj
@@ -1,4 +1,5 @@
 (ns redisload
+  (:import java.util.Calendar)
   (:use clojure.contrib.sql
         clojure.contrib.str-utils
         config
@@ -104,13 +105,58 @@ WHERE
          (redis/zadd datekey score msg-id))
        (println "inserted daily hall into" datekey)))))
 
-(println "streaming tags")
-(stream-tags [update-popular])
-(println (format "processed %s tags" @tag-counter))
 
-(redis/with-server redis-server
-  (transmit-favscores)
-  (transmit-popular)
-  (transmit-hall)
-  (transmit-daily-hall))
+(defn date-to-sql [dt]
+  (java.sql.Date. (.getTime dt)))
 
+(defn date-to-cal [dt]
+  (let [cal (Calendar/getInstance)]
+    (.setTime cal dt)
+    cal))
+
+(defn cal-to-date [cal]
+  (.getTime cal))
+
+(defn next-cal [cal]
+  (let [ret (.clone cal)]
+    (.add ret Calendar/DAY_OF_MONTH 1)
+    ret))
+
+(defn get-fav-date-range []
+  (let [{min :min max :max}
+        (first
+         (do-select ["select min(created_on)::date, max(created_on)::date from tags"]))
+        min (date-to-cal min)
+        max (date-to-cal max)]
+    (loop [acc  [(cal-to-date min)]
+           cur  min]
+      (let [next (next-cal cur)]
+        (if (.after cur max)
+          acc
+          (recur (conj acc (cal-to-date next)) next))))))
+    
+(defn get-top-posts [dt limit]
+  (do-select ["
+     SELECT m.message_id as message_id, count(*) as score
+     FROM messages m, tags t
+     WHERE m.message_id = t.message_id
+     AND m.created_on::date = ?::date
+     GROUP BY m.message_id
+     ORDER BY score DESC
+     LIMIT ?" (date-to-sql dt) limit]))
+
+(defn build-daily-hall [dt]
+  (let [top-posts (get-top-posts dt 200)
+        datekey   (redis-daily-hall-key (format-yyyymmdd dt))]
+    (println (count top-posts) "posts for" datekey)
+    (redis/with-server redis-server
+      (redis/atomically
+       (redis/del datekey)
+       (doseq [m top-posts]
+         (redis/zadd datekey (int (:score m)) (int (:message_id m))))))))
+
+(defn build-all-daily-hall []
+  (doseq [dt (get-fav-date-range)]
+    (build-daily-hall dt)))
+
+(build-all-daily-hall)
\ No newline at end of file