import { execFile } from 'child_process' import * as path from 'path' import * as fs from 'fs' import readdir from 'fs-readdir-promise' import sharp from 'sharp' import modules from './modules' import * as runner from './runner' const MEGABYTES = 1024 * 1024 const MAX_TRANSFER_SIZE = 2.5 * MEGABYTES const MAX_LOAD_SIZE = 108 * MEGABYTES const THUMBNAIL_SIZE = 200 const THUMBNAIL_QUALITY = 80 function sanitize_path(f){ return f.replace(/^\//,'').replace(/\.\./, '') } export function upload_file(task, cb) { const module = modules[task.module] const filepath = path.join(module.cwd, sanitize_path(task.path), sanitize_path(task.filename)) const params = [ '-F', 'module=' + task.module, '-F', 'activity=' + task.activity, '-F', 'generated=' + (String(task.generated) === 'true'), '-F', 'processed=' + (String(task.processed) === 'true'), '-F', "file=@" + filepath, process.env.API_REMOTE + '/api/folder/' + task.folder_id + '/upload/', ] console.log(params) execFile('curl', params, cb) // curl \ // -F "module=samplernn" \ // -F "activity=train" \ // -F "file=@woods1.jpg" \ // localhost:7013/api/folder/1/upload/ } export function status () { return { cpu: serialize_task(state.current_cpu_task), gpu: serialize_task(state.current_gpu_task), } } export function run_system_command(opt, cb) { console.log('running system command:', opt.cmd) switch(opt.cmd) { case 'nvidia-smi': case 'uptime': case 'w': execFile(opt.cmd, [], cb) break case 'ps': execFile('ps', ['au'], cb) break case 'df': execFile('df', ['-h'], cb) break case 'ls': list_directory(opt, cb) break case 'du': disk_usage(opt, cb) break case 'list_sequences': list_sequences(opt, cb) break case 'dir_to_video': dir_to_video(opt, cb) break default: cb({ error: 'no such command' }) break } } export function module_dir(opt, dir){ if (!opt.module || ! modules[opt.module]) { return null } const module = modules[opt.module] if (!module) { return null } return path.join(module.cwd, dir.replace(/\.\.?\//g, '')) } export function read_file(opt, cb) { const fn = module_dir(opt, opt.fn) if (!fn) return cb([]) stat_promise(fn).then(stat => { if (stat.size > MAX_TRANSFER_SIZE) { return cb({ error: 'file too large'}) } fs.readFile(fn, (err, buf) => cb({ error: err, name: opt.fn, path: fn, date: stat.ctime, size: stat.size, buf })) }).catch(() => cb({ error: 'error reading file' })) } export function thumbnail(opt, cb) { const fn = module_dir(opt, opt.fn) if (!fn) return cb([]) stat_promise(fn).then(stat => { if (stat.size > MAX_LOAD_SIZE) { return cb({ error: 'file too large'}) } sharp(fn) .resize(opt.size || THUMBNAIL_SIZE) .jpeg({ quality: opt.quality || THUMBNAIL_QUALITY }) .toBuffer() .then(buf => cb({ error: err, name: opt.fn, path: fn, date: stat.ctime, size: stat.size, buf })) .catch(err => cb({ error: err, name: opt.fn, path: fn, date: stat.ctime, size: stat.size, buf: null, })) }) } export function list_directory(opt, cb) { const dir = module_dir(opt, opt.dir) if (!dir) return cb([]) fs.readdir(dir, (err, files) => { const statPromises = (files || []).filter(f => f[0] !== '.').map(f => { return new Promise((resolve, reject) => { const full_path = path.join(dir, f) fs.stat(full_path, (err, stat={}) => { resolve({ name: f, date: stat.ctime, size: stat.size, dir: stat.isDirectory ? stat.isDirectory() : false, }) }) }) }) Promise.all(statPromises).then(stats => { cb(stats, dir) }).catch(error => { cb(error) }) }) } export function count_directory(opt, cb) { const dir = module_dir(opt, opt.dir) if (!dir) return cb([]) fs.readdir(dir, (err, files) => { err ? cb(err) : cb(files.length) }) } // list the contents of a directory of sequences export function list_sequences(opt, cb) { list_directory(opt, (files, root_dir) => { // console.log(files, root_dir) const sequencePromises = files.filter(d => !!d.dir).map(f => { return list_sequence(opt, f, root_dir) }) Promise.all(sequencePromises).then(cb).catch(error => { console.error(error) cb([]) }) }) } export function list_sequence(opt, f, root_dir) { return new Promise( (resolve, reject) => { let sequence = { name: f.name, date: f.date, size: f.size, frame: null, count: 0, } readdir(path.join(root_dir, f.name)).then(files => { if (! files.length) { return resolve(sequence) } const middle_file = files[Math.floor(files.length/2)] if (! middle_file) return resolve(sequence) sequence.frame = { prefix: middle_file.split('_')[0], } sequence.count = files.length // console.log(sequence.count) return stat_promise(path.join(root_dir, f.name, middle_file)) }).then(stat => { sequence.frame.date = stat.date sequence.frame.size = stat.size resolve(sequence) }).catch(err => reject(err)) }) } export function stat_promise(fn) { return new Promise((resolve, reject) => { fs.stat(fn, (err, stat={}) => { if (err) reject(err) resolve(stat) }) }) } export function dir_to_video(opt, db) { const dir = module_dir(opt, opt.dir) if (!dir) return cb([]) // input: the path (in results/) you want as a video // output: the path (in renders/) that contains the video // run the dir to video script with CWD as the directory and first input as ../renders plus the directory name // list the file in renders... execFile('./bin/dir_to_video.pl', params, { cwd: module.cwd, }, cb) } export function disk_usage(opt, cb) { const dir = module_dir(opt, opt.dir) if (!dir) return cb([]) execFile('du', ['-d', 1, dir], cb) } export function run_script(task, cb) { if (!task.module || ! modules[task.module]) { cb("") } const module = modules[task.module] const activity = module.activities[task.activity] const { interpreter, params, cancelled } = runner.build_params(module, activity, task) if (cancelled) return { type: 'error', error: "Task builder cancelled process" } if (! interpreter) return { type: 'error', error: "No such interpreter: " + activity.interpreter } if (! activity.isScript) return { type: 'error', error: "Not a script: " + task.module } console.log('running task', task.activity) console.log(module.cwd) console.log(interpreter.cmd, params) execFile(interpreter.cmd, params, { cwd: module.cwd, }, cb) }