split out system from runner

1 files changed, 258 insertions, 0 deletions

diff --git a/app/relay/system.js b/app/relay/system.js
new file mode 100644
index 0000000..90dbf66
--- /dev/null
+++ b/app/relay/system.js
@@ -0,0 +1,258 @@
+import { execFile } from 'child_process'
+import * as path from 'path'
+import * as fs from 'fs'
+import readdir from 'fs-readdir-promise'
+import sharp from 'sharp'
+
+import modules from './modules'
+import * as runner from './runner'
+
+const MEGABYTES = 1024 * 1024
+const MAX_TRANSFER_SIZE = 2.5 * MEGABYTES
+const MAX_LOAD_SIZE = 108 * MEGABYTES
+const THUMBNAIL_SIZE = 200
+const THUMBNAIL_QUALITY = 80
+
+function sanitize_path(f){
+  return f.replace(/^\//,'').replace(/\.\./, '')
+}
+
+export function upload_file(task, cb) {
+  const module = modules[task.module]
+  const filepath = path.join(module.cwd, sanitize_path(task.path), sanitize_path(task.filename))
+  const params = [
+    '-F', 'module=' + task.module,
+    '-F', 'activity=' + task.activity,
+    '-F', 'generated=' + (String(task.generated) === 'true'),
+    '-F', 'processed=' + (String(task.processed) === 'true'),
+    '-F', "file=@" + filepath,
+    process.env.API_REMOTE + '/api/folder/' + task.folder_id + '/upload/',
+  ]
+  console.log(params)
+  execFile('curl', params, cb)
+  // curl \
+  // -F "module=samplernn" \
+  // -F "activity=train" \
+  // -F "file=@woods1.jpg" \
+  // localhost:7013/api/folder/1/upload/
+}
+
+export function status () {
+  return {
+    cpu: serialize_task(state.current_cpu_task),
+    gpu: serialize_task(state.current_gpu_task),
+  }
+}
+
+export function run_system_command(opt, cb) {
+  console.log('running system command:', opt.cmd)
+  switch(opt.cmd) {
+    case 'nvidia-smi':
+    case 'uptime':
+    case 'w':
+      execFile(opt.cmd, [], cb)
+      break
+    case 'ps':
+      execFile('ps', ['au'], cb)
+      break
+    case 'df':
+      execFile('df', ['-h'], cb)
+      break
+    case 'ls':
+      list_directory(opt, cb)
+      break
+    case 'du':
+      disk_usage(opt, cb)
+      break
+    case 'list_sequences':
+      list_sequences(opt, cb)
+      break
+    case 'dir_to_video':
+      dir_to_video(opt, cb)
+      break
+    default:
+      cb({ error: 'no such command' })
+      break
+  }
+}
+
+export function module_dir(opt, dir){
+  if (!opt.module || ! modules[opt.module]) {
+    return null
+  }
+  const module = modules[opt.module]
+  if (!module) {
+    return null
+  }
+  return path.join(module.cwd, dir.replace(/\.\.?\//g, ''))
+}
+
+export function read_file(opt, cb) {
+  const fn = module_dir(opt, opt.fn)
+  if (!fn) return cb([])
+  stat_promise(fn).then(stat => {
+    if (stat.size > MAX_TRANSFER_SIZE) {
+      return cb({ error: 'file too large'})
+    }
+    fs.readFile(fn, (err, buf) => cb({
+      error: err,
+      name: opt.fn,
+      path: fn,
+      date: stat.ctime,
+      size: stat.size,
+      buf
+    }))
+  }).catch(() => cb({ error: 'error reading file' }))
+}
+
+export function thumbnail(opt, cb) {
+  const fn = module_dir(opt, opt.fn)
+  if (!fn) return cb([])
+  stat_promise(fn).then(stat => {
+    if (stat.size > MAX_LOAD_SIZE) {
+      return cb({ error: 'file too large'})
+    }
+    sharp(fn)
+      .resize(opt.size || THUMBNAIL_SIZE)
+      .jpeg({ quality: opt.quality || THUMBNAIL_QUALITY })
+      .toBuffer()
+      .then(buf => cb({
+        error: err,
+        name: opt.fn,
+        path: fn,
+        date: stat.ctime,
+        size: stat.size,
+        buf
+      }))
+      .catch(err => cb({
+        error: err,
+        name: opt.fn,
+        path: fn,
+        date: stat.ctime,
+        size: stat.size,
+        buf: null,
+      }))
+  })
+}
+
+export function list_directory(opt, cb) {
+  const dir = module_dir(opt, opt.dir)
+  if (!dir) return cb([])
+  fs.readdir(dir, (err, files) => {
+    const statPromises = (files || []).filter(f => f[0] !== '.').map(f => {
+      return new Promise((resolve, reject) => {
+        const full_path = path.join(dir, f)
+        fs.stat(full_path, (err, stat={}) => {
+          resolve({
+            name: f,
+            date: stat.ctime,
+            size: stat.size,
+            dir: stat.isDirectory ? stat.isDirectory() : false,
+          })
+        })
+      })
+    })
+    Promise.all(statPromises).then(stats => {
+      cb(stats, dir)
+    }).catch(error => {
+      cb(error)
+    })
+  })
+}
+
+export function count_directory(opt, cb) {
+  const dir = module_dir(opt, opt.dir)
+  if (!dir) return cb([])
+  fs.readdir(dir, (err, files) => {
+    err ? cb(err) : cb(files.length)
+  })
+}
+
+// list the contents of a directory of sequences
+export function list_sequences(opt, cb) {
+  list_directory(opt, (files, root_dir) => {
+    // console.log(files, root_dir)
+    const sequencePromises = files.filter(d => !!d.dir).map(f => {
+      return list_sequence(opt, f, root_dir)
+    })
+    Promise.all(sequencePromises).then(cb).catch(error => {
+      console.error(error)
+      cb([])
+    })
+  })
+}
+export function list_sequence(opt, f, root_dir) {
+  return new Promise( (resolve, reject) => {
+    let sequence = {
+      name: f.name,
+      date: f.date,
+      size: f.size,
+      frame: null,
+      count: 0,
+    }
+    readdir(path.join(root_dir, f.name)).then(files => {
+      if (! files.length) {
+        return resolve(sequence)
+      }
+      const middle_file = files[Math.floor(files.length/2)]
+      if (! middle_file) return resolve(sequence)
+      sequence.frame = {
+        prefix: middle_file.split('_')[0],
+      }
+      sequence.count = files.length
+      // console.log(sequence.count)
+      return stat_promise(path.join(root_dir, f.name, middle_file))
+    }).then(stat => {
+      sequence.frame.date = stat.date
+      sequence.frame.size = stat.size
+      resolve(sequence)
+    }).catch(err => reject(err))
+  })
+}
+
+export function stat_promise(fn) {
+  return new Promise((resolve, reject) => {
+    fs.stat(fn, (err, stat={}) => {
+      if (err) reject(err)
+      resolve(stat)
+    })
+  })
+}
+export function dir_to_video(opt, db) {
+  const dir = module_dir(opt, opt.dir)
+  if (!dir) return cb([])
+  // input: the path (in results/) you want as a video
+  // output: the path (in renders/) that contains the video
+  // run the dir to video script with CWD as the directory and first input as ../renders plus the directory name
+  // list the file in renders...
+  execFile('./bin/dir_to_video.pl', params, {
+    cwd: module.cwd,
+  }, cb)
+}
+
+
+export function disk_usage(opt, cb) {
+  const dir = module_dir(opt, opt.dir)
+  if (!dir) return cb([])
+  execFile('du', ['-d', 1, dir], cb)
+}
+
+export function run_script(task, cb) {
+  if (!task.module || ! modules[task.module]) {
+    cb("")
+  }
+  const module = modules[task.module]
+  const activity = module.activities[task.activity]
+  const { interpreter, params, cancelled } = runner.build_params(module, activity, task)
+  if (cancelled) return { type: 'error', error: "Task builder cancelled process" }
+  if (! interpreter) return { type: 'error', error: "No such interpreter: " + activity.interpreter }
+  if (! activity.isScript) return { type: 'error', error: "Not a script: " + task.module }
+
+  console.log('running task', task.activity)
+  console.log(module.cwd)
+  console.log(interpreter.cmd, params)
+  execFile(interpreter.cmd, params, {
+    cwd: module.cwd,
+  }, cb)
+}
+