From 180cf3eaa829fd04f759488d76641c4656caa245 Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Mon, 11 Apr 2016 14:20:33 -0400 Subject: rewrite file upload code to use knox directly --- app/node_modules/okserver/index.js | 2 + app/node_modules/okservices/oks3/index.js | 136 +++++++++++++++----------- app/node_modules/okservices/oks3/package.json | 5 +- app/node_modules/okservices/oks3/upload.js | 80 +++++++++++++++ 4 files changed, 162 insertions(+), 61 deletions(-) create mode 100644 app/node_modules/okservices/oks3/upload.js (limited to 'app/node_modules') diff --git a/app/node_modules/okserver/index.js b/app/node_modules/okserver/index.js index 3428565..302911d 100644 --- a/app/node_modules/okserver/index.js +++ b/app/node_modules/okserver/index.js @@ -52,6 +52,8 @@ function OKServer(options) { * other middleware. */ + // Disable silly express header + app.disable('x-powered-by') // Serve user static files app.use(express.static(root)); // Serve admin interface static files diff --git a/app/node_modules/okservices/oks3/index.js b/app/node_modules/okservices/oks3/index.js index 34c5840..c636308 100644 --- a/app/node_modules/okservices/oks3/index.js +++ b/app/node_modules/okservices/oks3/index.js @@ -1,5 +1,5 @@ -var skipper = require('skipper'); -var skipperS3 = require('skipper-s3') +var upload = require("./upload") +var multer = require('multer') // Hack to prevent this god-forsaken module from crashing our shit var d = require('domain').create() @@ -20,7 +20,6 @@ function OKS3(options) { if (!options.s3.audio) options.s3.audio = {} if (!options.s3.video) options.s3.video = {} -/* // TODO: maxBytes property doesn't work - if you upload a large file, // it will just hang until you reload the browser, and then CRASH. @@ -34,85 +33,104 @@ function OKS3(options) { if (! ('maxbytes' in options.s3.audio)) options.s3.audio.maxbytes = options.s3.maxbytes } -*/ - if (options.s3.image.preserveFilename) - options.s3.image.preserveFilename = preserveFilename - if (options.s3.video.preserveFilename) - options.s3.video.preserveFilename = preserveFilename - if (options.s3.audio.preserveFilename) - options.s3.audio.preserveFilename = preserveFilename + if (typeof options.s3.image.allowed !== "boolean") + options.s3.image.allowed = true + if (typeof options.s3.video.allowed !== "boolean") + options.s3.video.allowed = false + if (typeof options.s3.audio.allowed !== "boolean") + options.s3.audio.allowed = false + + upload.init({ + key: options.s3.key, + secret: options.s3.secret, + bucket: options.s3.bucket, + }) var express = options.express; var router = express.Router(); - router.use(skipper()); + var mult = multer() - // req should have a method `file` on it which is - // provided by skipper. Use that to do AWS stuff - router.post('/image', function(req, res) { + router.post('/image', mult.single('image'), function(req, res) { d.run(function () { - var skip = req.file('image').upload({ - adapter: skipperS3, - key: options.s3.key, - secret: options.s3.secret, - bucket: options.s3.bucket, + + if (! options.s3.image.allowed) { + return res.status(500).json({ error: "Image uploading not permitted" }) + } + + upload.put({ + file: req.file, + preserveFilename: options.s3.image.preserveFilename, dirname: options.s3.dirname, - // maxBytes: options.s3.image.maxbytes, - saveAs: options.s3.image.preserveFilename, - headers: { - 'x-amz-acl': 'public-read' + types: { + 'image/gif': 'gif', + 'image/jpeg': 'jpg', + 'image/jpg': 'jpg', + 'image/png': 'png', + }, + unacceptable: function(err){ + res.json({ error: err }) + }, + success: function(url){ + res.json({ url: url }) } - }, function (err, uploadedFiles) { - if (err) return res.status(500).send(err) - res.json(uploadedFiles); - }); + }) + }); }); - router.post('/audio', function(req, res) { + router.post('/audio', mult.single('audio'), function(req, res) { d.run(function () { - if (! options.s3.allowAudioUploads) { - return res.status(500).json({ error: "audio uploading not permitted" }) + + if (! options.s3.image.allowed) { + return res.status(500).json({ error: "Audio uploading not permitted" }) } - req.file('audio').upload({ - adapter: skipperS3, - key: options.s3.key, - secret: options.s3.secret, - bucket: options.s3.bucket, + + upload.put({ + file: req.file, + preserveFilename: options.s3.audio.preserveFilename, dirname: options.s3.dirname, - // maxBytes: options.s3.audio.maxbytes, - saveAs: options.s3.audio.preserveFilename, - headers: { - 'x-amz-acl': 'public-read' + types: { + 'audio/mp3': 'mp3', + 'audio/mpeg': 'mp3', + 'audio/wav': 'wav', + 'audio/flac': 'flac', + }, + unacceptable: function(err){ + res.json({ error: err }) + }, + success: function(url){ + res.json({ url: url }) } - }, function (err, uploadedFiles) { - if (err) res.status(500).send(err) - res.json(uploadedFiles); - }); + }) + }); }); - router.post('/video', function(req, res) { + router.post('/video', mult.single('video'), function(req, res) { d.run(function () { - if (! options.s3.allowVideoUploads) { - return res.status(500).json({ error: "video uploading not permitted" }) + + if (! options.s3.image.allowed) { + return res.status(500).json({ error: "Video uploading not permitted" }) } - req.file('video').upload({ - adapter: skipperS3, - key: options.s3.key, - secret: options.s3.secret, - bucket: options.s3.bucket, + + upload.put({ + file: req.file, + preserveFilename: options.s3.video.preserveFilename, dirname: options.s3.dirname, - // maxBytes: options.s3.video.maxbytes, - saveAs: options.s3.video.preserveFilename, - headers: { - 'x-amz-acl': 'public-read' + types: { + 'video/mp4': 'mp4', + 'video/webm': 'webm', + }, + unacceptable: function(err){ + res.json({ error: err }) + }, + success: function(url){ + res.json({ url: url }) } - }, function (err, uploadedFiles) { - if (err) res.status(500).send(err) - res.json(uploadedFiles); - }); + }) + }); }); diff --git a/app/node_modules/okservices/oks3/package.json b/app/node_modules/okservices/oks3/package.json index 19a2d2a..35bb4ce 100644 --- a/app/node_modules/okservices/oks3/package.json +++ b/app/node_modules/okservices/oks3/package.json @@ -9,7 +9,8 @@ "author": "OKFocus", "license": "None", "dependencies": { - "skipper": "^0.5.9", - "skipper-s3": "^0.5.6" + "knox": "^0.9.2", + "multer": "^1.1.0", + "node-uuid": "^1.4.7", } } diff --git a/app/node_modules/okservices/oks3/upload.js b/app/node_modules/okservices/oks3/upload.js new file mode 100644 index 0000000..6b16d61 --- /dev/null +++ b/app/node_modules/okservices/oks3/upload.js @@ -0,0 +1,80 @@ + +var knox = require('knox') +var moment = require('moment') +var uuid = require('node-uuid') + +var s3 + +var acceptableuploadTypes = { + 'image/gif': 'gif', + 'image/jpeg': 'jpg', + 'image/jpg': 'jpg', + 'image/png': 'png', +} + +module.exports = {} + +module.exports.init = function (opt){ + s3 = knox.createClient({ + key: opt.key, + secret: opt.secret, + bucket: opt.bucket, + }) +} + +module.exports.put = function (opt) { + var filename + var err + var now = new Date() + + var file = opt.file + + var types = opt.types || acceptableuploadTypes + var extension = types[file.mimetype] + + if (opt.preserveFilename) { + filename = file.originalname + } + else { + filename = uuid.v1() + "." + extension; + } + + var remote_path = "/" + opt.dirname + "/" + filename + + if (! extension) { + err = "Unacceptable filetype." + } + else if (opt.maxSize && file.size > opt.maxSize) { + err = "File too large. Uploads can be a maximum of " + opt.maxSize + " bytes." + } + + if (err) { + console.error(">>>", err) + opt.unacceptable && opt.unacceptable(err) + return + } + + opt.acceptable && opt.acceptable(err) + + // console.log("upload >", remote_path) + s3.putBuffer(file.buffer, remote_path, { + 'Content-Length': file.size, + 'Content-Type': file.mimetype, + 'x-amz-acl': 'public-read' + }, function(err, s3res) { + if (err || s3res.statusCode !== 200) { + console.error(err); + if (s3res && s3res.resume) { + s3res.resume() + } + return; + } + + var file_url = s3res.url || s3res.req.url + + opt.success && opt.success(file_url) + }).on('error', function(err, s3res){ + console.error(err) + s3res && s3res.resume && s3res.resume() + }) +} -- cgit v1.2.3-70-g09d2 From 3c33b364b4b3a9fef64a005713979f02fe04d4b9 Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Mon, 11 Apr 2016 14:49:06 -0400 Subject: draconian json parsing --- app/node_modules/okservices/oks3/package.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'app/node_modules') diff --git a/app/node_modules/okservices/oks3/package.json b/app/node_modules/okservices/oks3/package.json index 35bb4ce..61da414 100644 --- a/app/node_modules/okservices/oks3/package.json +++ b/app/node_modules/okservices/oks3/package.json @@ -11,6 +11,6 @@ "dependencies": { "knox": "^0.9.2", "multer": "^1.1.0", - "node-uuid": "^1.4.7", + "node-uuid": "^1.4.7" } } diff --git a/package.json b/package.json index 2d5df49..2a927d5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "okcms", - "version": "0.1.27", + "version": "0.1.30", "description": "great", "main": "app/index.js", "scripts": { -- cgit v1.2.3-70-g09d2 From 099c4091f1e98148e5ac05d1fc30cfd8333707bb Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Mon, 11 Apr 2016 14:58:07 -0400 Subject: typo --- app/node_modules/okservices/oks3/upload.js | 1 - 1 file changed, 1 deletion(-) (limited to 'app/node_modules') diff --git a/app/node_modules/okservices/oks3/upload.js b/app/node_modules/okservices/oks3/upload.js index 6b16d61..517d5f7 100644 --- a/app/node_modules/okservices/oks3/upload.js +++ b/app/node_modules/okservices/oks3/upload.js @@ -1,6 +1,5 @@ var knox = require('knox') -var moment = require('moment') var uuid = require('node-uuid') var s3 -- cgit v1.2.3-70-g09d2 From 5922279561c1998a3e158c26f951348f4c031928 Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Thu, 14 Apr 2016 14:17:00 -0400 Subject: support adding custom services per-project --- app/index.js | 42 ++++++++++++++++++------------- app/node_modules/okserver/index.js | 11 +++----- app/node_modules/okservices/oks3/index.js | 3 --- package.json | 5 ++-- themes/okadmin/public/js/upload.js | 3 --- 5 files changed, 32 insertions(+), 32 deletions(-) (limited to 'app/node_modules') diff --git a/app/index.js b/app/index.js index 1a8f97e..7fd88f4 100644 --- a/app/index.js +++ b/app/index.js @@ -94,19 +94,30 @@ function OKCMS(options) { errorHandler); // Create services - if (serviceConfig.s3) { - var s3Service = OKS3Service({ - express: express, - s3: serviceConfig.s3, - }); - } - - if (serviceConfig.twitter) { - var twitterService = OKTwitterService({ - express: express, - credentials: serviceConfig.twitter, - }) - } + var services = {} + Object.keys(serviceConfig).forEach(function(key){ + var config = serviceConfig[key] + switch (key) { + case 's3': + services.s3 = OKS3Service({ + express: express, + s3: config, + }); + break + case 'twitter': + services.twitter = OKTwitterService({ + express: express, + credentials: config, + }); + break + default: + services[key] = config.lib({ + express: express, + config: config, + }); + break + } + }); var server = this._server = new OKServer({ express: express, @@ -117,10 +128,7 @@ function OKCMS(options) { root: root, adminRoot: adminRoot, adminPath: adminPath, - services: { - s3: s3Service, - twitter: twitterService, - }, + services: services, errorHandler: errorHandler }); } diff --git a/app/node_modules/okserver/index.js b/app/node_modules/okserver/index.js index 302911d..a89676f 100644 --- a/app/node_modules/okserver/index.js +++ b/app/node_modules/okserver/index.js @@ -52,7 +52,7 @@ function OKServer(options) { * other middleware. */ - // Disable silly express header + // Disable x-powered-by express header app.disable('x-powered-by') // Serve user static files app.use(express.static(root)); @@ -61,12 +61,9 @@ function OKServer(options) { // Application router app.use(router); // Add services - if (services.s3) { - app.use('/_services/s3', services.s3.middleware()); - } - if (services.twitter) { - app.use('/_services/twitter', services.twitter.middleware()) - } + Object.keys(services).forEach(function(key){ + app.use('/_services/' + key, services[key].middleware()); + }) // Make sure this lady is last. Checks whether the desired // route has a trailing-slash counterpart and redirects there app.use(slash()); diff --git a/app/node_modules/okservices/oks3/index.js b/app/node_modules/okservices/oks3/index.js index c636308..cc40b71 100644 --- a/app/node_modules/okservices/oks3/index.js +++ b/app/node_modules/okservices/oks3/index.js @@ -20,9 +20,6 @@ function OKS3(options) { if (!options.s3.audio) options.s3.audio = {} if (!options.s3.video) options.s3.video = {} - // TODO: maxBytes property doesn't work - if you upload a large file, - // it will just hang until you reload the browser, and then CRASH. - // Make sure maxbytes property is there - it can be a number, // or zero/undefined (for no maximum upload size) if (options.s3.maxbytes) { diff --git a/package.json b/package.json index 2a927d5..bd5719b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "okcms", - "version": "0.1.30", + "version": "0.1.31", "description": "great", "main": "app/index.js", "scripts": { @@ -13,7 +13,8 @@ "dotenv": "^1.1.0", "express": "^4.12.3", "object-assign": "^2.0.0", - "q": "^1.2.0" + "q": "^1.2.0", + "request": "^2.71.0" }, "devDependencies": { "grunt": "^0.4.5", diff --git a/themes/okadmin/public/js/upload.js b/themes/okadmin/public/js/upload.js index da23b1b..6ff7ac9 100644 --- a/themes/okadmin/public/js/upload.js +++ b/themes/okadmin/public/js/upload.js @@ -177,9 +177,6 @@ OKUpload.prototype.add = function(media){ OKUpload.prototype.addMedia = function(media){ console.log(media) } -OKUpload.prototype.addAudio = function(media){ - console.log(media) -} OKUpload.prototype.error = function(error){ throw error } -- cgit v1.2.3-70-g09d2 From c98560cca095ef27b909ac47baf843362bffb28d Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Thu, 14 Apr 2016 20:32:08 -0400 Subject: OKWebhook module for receiving pushes from git --- app/index.js | 7 ++ app/node_modules/okservices/okwebhook/index.js | 83 ++++++++++++++++++++++ app/node_modules/okservices/okwebhook/package.json | 11 +++ examples/index.js | 11 ++- examples/lib/okexample/index.js | 5 +- 5 files changed, 114 insertions(+), 3 deletions(-) create mode 100644 app/node_modules/okservices/okwebhook/index.js create mode 100644 app/node_modules/okservices/okwebhook/package.json (limited to 'app/node_modules') diff --git a/app/index.js b/app/index.js index 8c3abb9..48e713a 100644 --- a/app/index.js +++ b/app/index.js @@ -15,6 +15,7 @@ var OKServer = require('okserver'); var OKSchema = require('okschema'); var OKS3Service = require('okservices/oks3'); var OKTwitterService = require('okservices/oktwitter') +var OKWebhookService = require('okservices/okwebhook') require('dotenv').load(); @@ -110,6 +111,12 @@ function OKCMS(options) { credentials: config, }); break + case 'webhook': + services.webhook = OKWebhookService({ + express: express, + config: config, + }); + break default: services[key] = config.lib({ db: resourceCache, diff --git a/app/node_modules/okservices/okwebhook/index.js b/app/node_modules/okservices/okwebhook/index.js new file mode 100644 index 0000000..d04e662 --- /dev/null +++ b/app/node_modules/okservices/okwebhook/index.js @@ -0,0 +1,83 @@ + +/** + * Service which will listen for a Github webhook, fired on push. + * This service can be used to rebuild / restart the app automatically + * when new code is pushed. + */ + +var crypto = require('crypto') +var exec = require('child_process').exec +var path = require('path') + +function OKWebhook (options) { + if (!(this instanceof OKWebhook)) return new OKWebhook(options) + options = options || {} + if (!options.express) + throw new Error('Express not provided to OKWebhook'); + if (!options.config) + throw new Error('Configuration not provided to OKWebhook'); + if (options.config.active && !options.config.secret) + throw new Error('Github secret not provided to OKWebhook'); + if (options.config.active && !options.config.command) + throw new Error('Build command not provided to OKWebhook'); + + var express = options.express + var router = express.Router() + var config = options.config + + var secret = config.secret + var command = config.command + + router.get('/', function (req, res) { + res.send('GET not supported') + }) + + router.post('/', getBody, function (req, res) { + if (!config.active) + return + console.log("OKWebhook received push") + var event = req.headers['x-github-event'] + if (event !== "push") { + return res.sendStatus(500) + } + var sig = req.headers['x-hub-signature'].split('=')[1] + var text = req.rawBody + var hash = crypto.createHmac('sha1', secret).update(text).digest('hex') + if (hash !== sig) { + return res.sendStatus(500) + } + res.sendStatus(200) + var cwd = path.dirname(command) + exec(command, { cwd: cwd }, function(err, stdout, stderr){ + // may not fire if process was restarted.. + console.log(process.env) + console.log(stdout) + }) + }) + + function getBody (req, res, next) { + req.rawBody = '' + // req.setEncoding('utf8') + + req.on('data', function(chunk) { + req.rawBody += chunk + }) + + req.on('end', function() { + try { + req.body = JSON.parse(req.rawBody) + } catch (e) { + return res.sendStatus(500) + } + next() + }) + } + + this._router = router +} + +OKWebhook.prototype.middleware = function () { + return this._router +} + +module.exports = OKWebhook diff --git a/app/node_modules/okservices/okwebhook/package.json b/app/node_modules/okservices/okwebhook/package.json new file mode 100644 index 0000000..0436f01 --- /dev/null +++ b/app/node_modules/okservices/okwebhook/package.json @@ -0,0 +1,11 @@ +{ + "name": "okwebhook", + "version": "1.0.0", + "description": "webhook to receive pushes from github", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "okfocus ", + "license": "LNT" +} diff --git a/examples/index.js b/examples/index.js index 3bbed28..efbfc38 100644 --- a/examples/index.js +++ b/examples/index.js @@ -1,10 +1,13 @@ var okcms = require('..'); +var isProduction = process.env.OK_PRODUCTION === 'true' + var app = okcms.createApp({ root: 'public', - debug: true, + debug: !isProduction, + production: isProduction, schemas: { page: { @@ -47,6 +50,12 @@ var app = okcms.createApp({ audio: { allowed: true, preserveFilename: true, maxbytes: 100*1024*1024 }, }, + webhook: { + active: false, + secret: 'test', + command: '/path/to/build.sh', + }, + example: { lib: require("./lib/okexample"), stuff: "things", diff --git a/examples/lib/okexample/index.js b/examples/lib/okexample/index.js index b614697..04c5984 100644 --- a/examples/lib/okexample/index.js +++ b/examples/lib/okexample/index.js @@ -22,13 +22,14 @@ function OKExample (options) { if (!(this instanceof OKExample)) return new OKExample(options) options = options || {} if (!options.express) - throw new Error('Express not provided to OKDumpfm'); + throw new Error('Express not provided to OKExample'); if (!options.config) - throw new Error('Configuration not provided to OKDumpfm'); + throw new Error('Configuration not provided to OKExample'); var express = options.express var router = express.Router() var config = options.config + var db = options.db router.get('*', function (req, res) { res.send(config.stuff) -- cgit v1.2.3-70-g09d2