var fetch = require('node-fetch') const fs = require('fs') var db = require('../db') var upload = require('../util/upload') var mime = require('mime-types') module.exports = { route: (app) => { app.put('/raw/import/thread/', importRaw('thread', 'Thread'), (req, res) => res.send({ status: 'ok', el: res.el })) app.put('/raw/import/keyword/', importRaw('keyword', 'Keyword'), (req, res) => res.send({ status: 'ok', el: res.el })) app.put('/raw/import/file/', importRaw('file', 'File'), (req, res) => res.send({ status: 'ok', el: res.el })) app.put('/raw/import/comment/', importRaw('comment', 'Comment'), (req, res) => res.send({ status: 'ok', el: res.el })) app.get('/raw/export/thread/:id', exportThread, (req, res) => res.send({ status: 'ok' })) app.get('/raw/export/keyword/:keyword', exportKeyword, (req, res) => res.send({ status: 'ok' })) function importRaw (type, model) { return (req, res, next) => { console.log('importing', type, req.body.id) delete req.body.id db[model].forge(req.body).save().then((el) => { res.el = el; next() }).catch(e => { console.error(e) next() }) } } function exportKeyword (req, res, next) { console.log('export keyword', req.params.keyword) db.getKeyword(req.params.keyword).then(keyword => { send("keyword", keyword) return db.getThreadsForKeyword(req.params.keyword) }).then(threads => { var promises = threads.map(thread => { exportThread({ params: { id: thread.get('id') } }, res, function(){}) }) return Promise.all(promises) }).then( () => { next() }) } function exportThread (req, res, next) { var thread_id return db.getThread(req.params.id).then(thread => { return send("thread", thread) }).then(json => { thread_id = json.el.id console.log('got thread id', thread_id) return db.getCommentsForThread(req.params.id) }).then(comments => { var promises = comments.map(comment => { comment.set('thread', thread_id) send("comment", comment) }) return Promise.all(promises) }).then( () => { return db.getFilesForThread(req.params.id) }).then(files => { var promises = files.map(file => { copyFileToS3(file, thread_id) file.set('thread', thread_id) file.set('storage', 'i.asdf.us') return send("file", file) }) return promises }).then( () => { next() }).catch(e => { console.error(e) next() }) } function send(type, data){ console.log('sending', type, data.get('id')) var json = data.toJSON() return fetch("https://bucky.asdf.us/raw/import/" + type, { method: 'PUT', body: JSON.stringify(json), headers: { 'Content-Type': 'application/json', 'Accept': 'application/json', }, }).then((res) => {return res.json()}) then((json) => console.log(json)) } function copyFileToS3(file, thread_id){ // since for now we are essentially backing up local files, // upload them directly to s3 const src_fn = file.get('thread') + '/' + file.get('filename') const dst_fn = thread_id + '/' + file.get('filename') fs.readFile('/Users/user/projects/bucky3/public/data/' + src_fn, (err, buffer) => { if (err) return console.log(err) const remote_path = '/bucky/data/' + dst_fn console.log(mime.lookup(file.get('filename'))) upload.client().putBuffer(buffer, remote_path, { 'Content-Length': buffer.length, 'Content-Type': mime.lookup(file.get('filename')), 'x-amz-acl': 'public-read' }, function(err, s3res) { if (err || s3res.statusCode !== 200) { console.error(err); if (s3res && s3res.resume) { s3res.resume() } return; } var file_url = s3res.url || s3res.req.url console.log(file_url) }).on('error', function(err, s3res){ console.error(err) s3res && s3res.resume && s3res.resume() }) }) } } }