split out system from runner

1 files changed, 1 insertions, 253 deletions

diff --git a/app/relay/runner.js b/app/relay/runner.js
index 1e90fca..70988b0 100644
--- a/app/relay/runner.js
+++ b/app/relay/runner.js
@@ -1,22 +1,13 @@
-import { execFile, spawn } from 'child_process'
+import { spawn } from 'child_process'
 import interpreters from './interpreters'
 import modules from './modules'
 import kill from 'tree-kill'
 import { remote } from './remote'
 import { set_connected } from './rpc'
 import uuidv1 from 'uuid/v1'
-import * as fs from 'fs'
-import * as path from 'path'
-import readdir from 'fs-readdir-promise'
-import sharp from 'sharp'
 
 import * as q from './queue'
 
-const MEGABYTES = 1024 * 1024
-const MAX_TRANSFER_SIZE = 2.5 * MEGABYTES
-const MAX_LOAD_SIZE = 108 * MEGABYTES
-const THUMBNAIL_SIZE = 200
-const THUMBNAIL_QUALITY = 80
 const idle_state = { status: 'IDLE', task: {} }
 
 export const state = {
@@ -68,37 +59,6 @@ function clear_task(is_gpu, task){
   }
 }
 
-function sanitize_path(f){
-  return f.replace(/^\//,'').replace(/\.\./, '')
-}
-
-export function upload_file(task, cb) {
-  const module = modules[task.module]
-  const filepath = path.join(module.cwd, sanitize_path(task.path), sanitize_path(task.filename))
-  const params = [
-    '-F', 'module=' + task.module,
-    '-F', 'activity=' + task.activity,
-    '-F', 'generated=' + (String(task.generated) === 'true'),
-    '-F', 'processed=' + (String(task.processed) === 'true'),
-    '-F', "file=@" + filepath,
-    process.env.API_REMOTE + '/api/folder/' + task.folder_id + '/upload/',
-  ]
-  console.log(params)
-  execFile('curl', params, cb)
-  // curl \
-  // -F "module=samplernn" \
-  // -F "activity=train" \
-  // -F "file=@woods1.jpg" \
-  // localhost:7013/api/folder/1/upload/
-}
-
-export function status () {
-  return {
-    cpu: serialize_task(state.current_cpu_task),
-    gpu: serialize_task(state.current_gpu_task),
-  }
-}
-
 export function build_params(module, activity, task) {
 	const interpreter = interpreters[activity.type]
   let opt_params, activity_params;
@@ -132,218 +92,6 @@ export function build_params(module, activity, task) {
 	}
 }
 
-export function run_system_command(opt, cb) {
-  console.log('running system command:', opt.cmd)
-  switch(opt.cmd) {
-    case 'nvidia-smi':
-    case 'uptime':
-    case 'w':
-      execFile(opt.cmd, [], cb)
-      break
-    case 'ps':
-      execFile('ps', ['au'], cb)
-      break
-    case 'df':
-      execFile('df', ['-h'], cb)
-      break
-    case 'ls':
-      list_directory(opt, cb)
-      break
-    case 'du':
-      disk_usage(opt, cb)
-      break
-    case 'list_sequences':
-      list_sequences(opt, cb)
-      break
-    case 'dir_to_video':
-      dir_to_video(opt, cb)
-      break
-    default:
-    	cb({ error: 'no such command' })
-    	break
-  }
-}
-
-export function module_dir(opt, dir){
-  if (!opt.module || ! modules[opt.module]) {
-    return null
-  }
-  const module = modules[opt.module]
-  if (!module) {
-    return null
-  }
-  return path.join(module.cwd, dir.replace(/\.\.?\//g, ''))
-}
-
-export function read_file(opt, cb) {
-  const fn = module_dir(opt, opt.fn)
-  if (!fn) return cb([])
-  stat_promise(fn).then(stat => {
-    if (stat.size > MAX_TRANSFER_SIZE) {
-      return cb({ error: 'file too large'})
-    }
-    fs.readFile(fn, (err, buf) => cb({
-      error: err,
-      name: opt.fn,
-      path: fn,
-      date: stat.ctime,
-      size: stat.size,
-      buf
-    }))
-  }).catch(() => cb({ error: 'error reading file' }))
-}
-
-export function thumbnail(opt, cb) {
-  const fn = module_dir(opt, opt.fn)
-  if (!fn) return cb([])
-  stat_promise(fn).then(stat => {
-    if (stat.size > MAX_LOAD_SIZE) {
-      return cb({ error: 'file too large'})
-    }
-    sharp(fn)
-      .resize(opt.size || THUMBNAIL_SIZE)
-      .jpeg({ quality: opt.quality || THUMBNAIL_QUALITY })
-      .toBuffer()
-      .then(buf => cb({
-        error: err,
-        name: opt.fn,
-        path: fn,
-        date: stat.ctime,
-        size: stat.size,
-        buf
-      }))
-      .catch(err => cb({
-        error: err,
-        name: opt.fn,
-        path: fn,
-        date: stat.ctime,
-        size: stat.size,
-        buf: null,
-      }))
-  })
-}
-
-export function list_directory(opt, cb) {
-  const dir = module_dir(opt, opt.dir)
-  if (!dir) return cb([])
-  fs.readdir(dir, (err, files) => {
-    const statPromises = (files || []).filter(f => f[0] !== '.').map(f => {
-      return new Promise((resolve, reject) => {
-        const full_path = path.join(dir, f)
-        fs.stat(full_path, (err, stat={}) => {
-          resolve({
-            name: f,
-            date: stat.ctime,
-            size: stat.size,
-            dir: stat.isDirectory ? stat.isDirectory() : false,
-          })
-        })
-      })
-    })
-    Promise.all(statPromises).then(stats => {
-      cb(stats, dir)
-    }).catch(error => {
-      cb(error)
-    })
-  })
-}
-
-export function count_directory(opt, cb) {
-  const dir = module_dir(opt, opt.dir)
-  if (!dir) return cb([])
-  fs.readdir(dir, (err, files) => {
-    err ? cb(err) : cb(files.length)
-  })
-}
-
-// list the contents of a directory of sequences
-export function list_sequences(opt, cb) {
-  list_directory(opt, (files, root_dir) => {
-    // console.log(files, root_dir)
-    const sequencePromises = files.filter(d => !!d.dir).map(f => {
-      return list_sequence(opt, f, root_dir)
-    })
-    Promise.all(sequencePromises).then(cb).catch(error => {
-      console.error(error)
-      cb([])
-    })
-  })
-}
-export function list_sequence(opt, f, root_dir) {
-  return new Promise( (resolve, reject) => {
-    let sequence = {
-      name: f.name,
-      date: f.date,
-      size: f.size,
-      frame: null,
-      count: 0,
-    }
-    readdir(path.join(root_dir, f.name)).then(files => {
-      if (! files.length) {
-        return resolve(sequence)
-      }
-      const middle_file = files[Math.floor(files.length/2)]
-      if (! middle_file) return resolve(sequence)
-      sequence.frame = {
-        prefix: middle_file.split('_')[0],
-      }
-      sequence.count = files.length
-      // console.log(sequence.count)
-      return stat_promise(path.join(root_dir, f.name, middle_file))
-    }).then(stat => {
-      sequence.frame.date = stat.date
-      sequence.frame.size = stat.size
-      resolve(sequence)
-    }).catch(err => reject(err))
-  })
-}
-
-export function stat_promise(fn) {
-  return new Promise((resolve, reject) => {
-    fs.stat(fn, (err, stat={}) => {
-      if (err) reject(err)
-      resolve(stat)
-    })
-  })
-}
-export function dir_to_video(opt, db) {
-  const dir = module_dir(opt, opt.dir)
-  if (!dir) return cb([])
-  // input: the path (in results/) you want as a video
-  // output: the path (in renders/) that contains the video
-  // run the dir to video script with CWD as the directory and first input as ../renders plus the directory name
-  // list the file in renders...
-  execFile('./bin/dir_to_video.pl', params, {
-    cwd: module.cwd,
-  }, cb)
-}
-
-
-export function disk_usage(opt, cb) {
-  const dir = module_dir(opt, opt.dir)
-  if (!dir) return cb([])
-  execFile('du', ['-d', 1, dir], cb)
-}
-
-export function run_script(task, cb) {
-  if (!task.module || ! modules[task.module]) {
-    cb("")
-  }
-  const module = modules[task.module]
-  const activity = module.activities[task.activity]
-  const { interpreter, params, cancelled } = build_params(module, activity, task)
-  if (cancelled) return { type: 'error', error: "Task builder cancelled process" }
-  if (! interpreter) return { type: 'error', error: "No such interpreter: " + activity.interpreter }
-  if (! activity.isScript) return { type: 'error', error: "Not a script: " + task.module }
-
-  console.log('running task', task.activity)
-  console.log(module.cwd)
-  console.log(interpreter.cmd, params)
-  execFile(interpreter.cmd, params, {
-    cwd: module.cwd,
-  }, cb)
-}
-
 export function get_processor(task){
   if (! task) return null
   const module = modules[task.module]