moved search to redis, no more bdb

7 files changed, 102 insertions, 80 deletions

diff --git a/bucky/app/api.js b/bucky/app/api.js
index b4231e7..75cdbd7 100644
--- a/bucky/app/api.js
+++ b/bucky/app/api.js
@@ -163,8 +163,8 @@ function route (app){
     privacy.checkThreadPrivacy,
     multer.array("files"),
     bucky.verifyFilesOrComment,
-    bucky.createOptionalFiles,
     bucky.createOptionalComment,
+    bucky.createOptionalFiles,
     bucky.bumpThreadRevisions,
     function(req, res){
       res.json({
diff --git a/bucky/app/bucky.js b/bucky/app/bucky.js
index 3824c5e..8d9839f 100644
--- a/bucky/app/bucky.js
+++ b/bucky/app/bucky.js
@@ -413,6 +413,7 @@ var bucky = module.exports = {
       res.json({ error: "no comment" })
       return
     }
+    console.log("in create comment")
     var data = {
       thread: res.thread.get('id'),
       parent_id: req.body.parent_id || -1,
@@ -503,9 +504,15 @@ var bucky = module.exports = {
               privacy: false,
               storage: process.env.S3_BUCKET,
             }
+            req.body.comment = url+"\n"+file.originalname
             db.createFile(data).then(function(file){
               resolve(file)
-            }).catch( (err) => reject(err) )
+            }).catch( (err) => reject(err) ).then(
+            function(){
+              console.log("about to call createComment")
+              bucky.createComment(req, res, function(){})
+
+            })
           }
         })
       })
diff --git a/bucky/app/site.js b/bucky/app/site.js
index d38b094..0d35fd6 100644
--- a/bucky/app/site.js
+++ b/bucky/app/site.js
@@ -13,7 +13,10 @@ var favicon = require('serve-favicon')
 var passport = require('passport')
 var sessionstore = require('sessionstore')
 var session = require('express-session')
-var MongoStore = require('connect-mongo')(session);
+var redis = require('redis')
+var RedisStore = require('connect-redis')(session)
+var redisClient = redis.createClient()
+
 var upload = require('../util/upload')
 
 var app, server
@@ -45,16 +48,10 @@ site.init = function(){
     saveUninitialized: false,
   }
   if (!process.env.SESSIONS_IN_MEMORY) {
-    sessionSettings.store = new MongoStore({
-      url: 'mongodb://127.0.0.1:28108/buckySessionDb-'+ process.env.DB_NAME
-      // type: 'mongodb',
-      // host: 'localhost',
-      // port: 27017,
-      // dbName: 'buckySessionDb',
-      // collectionName: 'sessions',
-      // timeout: 10000,
-    })
+      sessionSettings.store =
+        new RedisStore({ client: redisClient })
   }
+
   app.use(session(sessionSettings))
   upload.init()
   federate.route(app)
diff --git a/bucky/search/lexicon.js b/bucky/search/lexicon.js
index dc1d7ab..ea99535 100644
--- a/bucky/search/lexicon.js
+++ b/bucky/search/lexicon.js
@@ -1,10 +1,8 @@
 require('dotenv').load();
 
 var STOPWORDS = require('./stopwords')
-var bdb = require('./bdb')
 var db = require('../db')
-
-var search_db = bdb('search')
+var redisClient = require('./redis-client');
 
 var lexicon = {}
 var lex_counts = {}
@@ -26,6 +24,7 @@ function build_index(cb) {
       return { total, unique }
     })
 }
+
 function parse_threads() {
   return db.Thread.where('id', '>', 1).fetchAll().then( (threads) => {
     console.log('got threads', threads.length)
@@ -41,8 +40,7 @@ function parse_comments() {
   return db.Comment.where('thread', '>', 1).fetchAll().then( (comments) => {
     console.log('got comments', comments.length)
     comments.forEach( (comment) => {
-      total += parse_terms({
-        string: comment.get('comment').toString(),
+      total += parse_terms({ string: comment.get('comment').toString(),
         thread: comment.get('thread'),
         comment: comment.get('id'),
       })
@@ -107,7 +105,7 @@ function lexicon_store () {
 		put_total += 1
 		// if (put_total > 10) return
 		// console.log(term)
-		search_db.put(term, serialized)
+    redisClient.set(term, serialized)
   })
 }
 function serialize_matches (term) {
@@ -126,4 +124,4 @@ function serialize_matches (term) {
   })
 	if (!serialized_matches.length) return
 	return serialized_matches.join(',')
-}
\ No newline at end of file
+}
diff --git a/bucky/search/middleware.js b/bucky/search/middleware.js
index 0cca05c..58ac4c8 100644
--- a/bucky/search/middleware.js
+++ b/bucky/search/middleware.js
@@ -7,12 +7,19 @@ var lexicon = require('./lexicon')
 module.exports = {
   
   search: function (req, res, next) {
-    res.search = search.search(req.query.query, req.query.start, req.query.limit)
-    if (! res.search) {
-      res.sendStatus(400)
-      return
-    }
-    next()
+    search.search(
+      req.query.query, 
+      req.query.start, 
+      req.query.limit,
+      function(search_results){
+         res.search = search_results
+         if (! res.search) {
+           res.sendStatus(400)
+           return
+         }
+         next()
+      }
+    )
   },
   
   getThreads: function (req, res, next){
diff --git a/bucky/search/search.js b/bucky/search/search.js
index 1a56f53..bde1ad9 100644
--- a/bucky/search/search.js
+++ b/bucky/search/search.js
@@ -1,5 +1,8 @@
 var db = require('../db')
-var bdb = require('./bdb')('search')
+var eachSeries = require('async/eachSeries');
+var redisClient = require('./redis-client.js')
+const { promisify } = require("util");
+const lookupAsync = promisify(redisClient.get).bind(redisClient);
 var STOPWORDS = require('./stopwords')
 
 var wordRegexp = new RegExp("[^a-z0-9]+", 'g');
@@ -8,11 +11,9 @@ function parse_terms (s) {
 }
 function cmp (a,b){ return (a<b)?-1:(a===b)?0:1 }
 
-function find_term(term) {
-  var res = bdb.get(term).toString()
-  // console.log(res)
-  if (! res.length) return []
-  var matches = res.split(",").map((s) => {
+function split_results(data) {
+ if (! data.length) return []
+  var matches = data.split(",").map((s) => {
     if (! s.length) return;
     var partz = s.split(" ")
     return {
@@ -22,11 +23,11 @@ function find_term(term) {
       strength: parseInt(partz[3]) || 1,
     }
   })
-  // console.log(matches)
   return matches
 }
 
-function search (query, start, limit) {
+function search (query, start, limit, cb) {
+  
   if (!query) return
 	start = parseInt(start) || 0;
 	limit = parseInt(limit) || 10;
@@ -40,50 +41,62 @@ function search (query, start, limit) {
   var file_ids = []
   var results = []
 
-  terms.forEach((term) => {
-    if (STOPWORDS.has(term)) return;
-    var results = find_term(term);
-    if (!results) return;
-    results.forEach((result) => {
-      var score = scores[result.thread] = scores[result.thread] || { count: 0, strength: 0 }
-      score.thread = score.thread || parseInt(result.thread)
-      score.comment = score.comment || parseInt(result.comment)
-      score.file = score.file || parseInt(result.file)
-      score.strength += parseFloat(result.strength)
-      score.count += 1
-    })
-  })
-  total = Object.keys(scores).length
-  Object.values(scores).sort((b,a) => {
-    // if (a.count !== b.count) {
-    //   return cmp(a.count, b.count)
-    // }
-    return cmp(a.strength, b.strength)
-  }).some((match, i) => {
-    if (i < start) return false
-    if (to_display-- === 0) return true
-    results.push(match)
-    // console.log(match)
-    thread_ids.push(match.thread)
-    if (match.comment) comment_ids.push(match.comment)
-    if (match.file) file_ids.push(match.file)
-    return false
+
+  console.log(terms)
+  eachSeries(
+    terms, 
+    function(term, callback){
+      console.log(term);
+      if (STOPWORDS.has(term)) return;
+      console.log("howdy")
+      redisClient.get(term, function(err, results){
+//        if (!results) return callback();
+
+        results = split_results(results)
+        results.forEach((result) => {
+          //this is a reference, mutating scores
+          var score = scores[result.thread] = scores[result.thread] || { count: 0, strength: 0 }
+          score.thread = score.thread || parseInt(result.thread)
+          score.comment = score.comment || parseInt(result.comment)
+          score.file = score.file || parseInt(result.file)
+          score.strength += parseFloat(result.strength)
+          score.count += 1
+        })
+        callback()
+      })
+     }, 
+    function() {
+      total = Object.keys(scores).length
+      Object.values(scores).sort((b,a) => {
+        return cmp(a.strength, b.strength)
+      }).some((match, i) => {
+        if (i < start) return false
+        if (to_display-- === 0) return true
+        results.push(match)
+        // console.log(match)
+        thread_ids.push(match.thread)
+        if (match.comment) comment_ids.push(match.comment)
+        if (match.file) file_ids.push(match.file)
+        return false
+      })
+      //console.log(results)
+      redisClient.quit()
+      cb( {
+        meta: {
+          query: query,
+          terms: terms,
+          start: start,
+          next: start + limit,
+          limit: limit,
+          total: total,
+        },
+        results: results,
+        thread_ids: thread_ids,
+        comment_ids: comment_ids,
+        file_ids: file_ids,
+      });
   })
 
-  return {
-    meta: {
-      query: query,
-      terms: terms,
-      start: start,
-      next: start + limit,
-      limit: limit,
-      total: total,
-    },
-    results: results,
-		thread_ids: thread_ids,
-		comment_ids: comment_ids,
-		file_ids: file_ids,
-  };
 }
 
 module.exports = { search: search }
diff --git a/bucky/util/auth.js b/bucky/util/auth.js
index d457704..709e591 100644
--- a/bucky/util/auth.js
+++ b/bucky/util/auth.js
@@ -24,11 +24,11 @@ var auth = module.exports = {
           title: "login"
         })
       })
-    app.get("/signup", function(req, res){
-      res.render("pages/signup", {
-        title: "signup"
-      })
-    })
+//    app.get("/signup", function(req, res){
+//      res.render("pages/signup", {
+//        title: "signup"
+//      })
+//    })
     app.get("/logout", auth.logout)
 
     app.put("/api/signup",