From 964ac7009e6db5a06233bdc07fa63778eebf2db7 Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Fri, 1 Jun 2018 03:30:39 +0200 Subject: async commands!! --- app/client/actions.js | 24 +++++ app/client/api/crud.actions.js | 14 ++- app/client/api/crud.upload.js | 4 + app/client/common/fileList.component.js | 13 ++- app/client/dashboard/dashboardHeader.component.js | 36 +++++-- app/client/dataset/dataset.reducer.js | 110 +++------------------ app/client/modules/samplernn/samplernn.actions.js | 40 ++++++++ app/client/modules/samplernn/samplernn.datasets.js | 27 +---- app/client/modules/samplernn/samplernn.reducer.js | 48 ++++++++- app/client/socket/socket.actions.js | 3 + app/client/socket/socket.system.js | 49 +++++++-- app/client/system/system.actions.js | 5 + app/client/types.js | 2 + app/client/util.js | 21 ++++ app/relay/remote.js | 20 +++- app/relay/runner.js | 30 ++++++ 16 files changed, 306 insertions(+), 140 deletions(-) create mode 100644 app/client/actions.js create mode 100644 app/client/socket/socket.actions.js (limited to 'app') diff --git a/app/client/actions.js b/app/client/actions.js new file mode 100644 index 0000000..ff170bb --- /dev/null +++ b/app/client/actions.js @@ -0,0 +1,24 @@ +import { bindActionCreators } from 'redux' +import { actions as crudActions } from './api' +import * as taskActions from './task/task.actions' +import * as liveActions from './live/live.actions' +import * as systemActions from './system/system.actions' +import * as socketActions from './socket/socket.actions' +import * as datasetActions from './dataset/dataset.actions' + +import { dispatch } from './store' + +export default + Object.keys(crudActions) + .map(a => [a, crudActions[a]]) + .concat([ + ['task', taskActions], + ['live', liveActions], + ['system', systemActions], + ['dataset', datasetActions], + ]) + .map(p => [p[0], bindActionCreators(p[1], dispatch)]) + .concat([ + ['socket', socketActions], + ]) + .reduce((a,b) => (a[b[0]] = b[1])&&a,{}) \ No newline at end of file diff --git a/app/client/api/crud.actions.js b/app/client/api/crud.actions.js index 1fcae81..f8b3bac 100644 --- a/app/client/api/crud.actions.js +++ b/app/client/api/crud.actions.js @@ -20,10 +20,14 @@ export function crud_actions(type) { } export const crud_action = (type, method, fn) => q => dispatch => { - dispatch({ type: as_type(type, method + '_loading') }) - fn(q).then(data => { - dispatch({ type: as_type(type, method), data }) - }).catch(e => { - dispatch({ type: as_type(type, method + '_error') }) + return new Promise ((resolve, reject) => { + dispatch({ type: as_type(type, method + '_loading') }) + fn(q).then(data => { + dispatch({ type: as_type(type, method), data }) + resolve(data) + }).catch(e => { + dispatch({ type: as_type(type, method + '_error') }) + reject(e) + }) }) } diff --git a/app/client/api/crud.upload.js b/app/client/api/crud.upload.js index 65ae4e0..01c3e18 100644 --- a/app/client/api/crud.upload.js +++ b/app/client/api/crud.upload.js @@ -57,6 +57,7 @@ export function crud_upload(type, fd, data, dispatch) { error: 'upload failed', [type]: id, }) + reject(e) return } dispatch && dispatch({ @@ -64,6 +65,7 @@ export function crud_upload(type, fd, data, dispatch) { data, [type]: id, }) + resolve(data) } function uploadFailed (evt) { @@ -72,6 +74,7 @@ export function crud_upload(type, fd, data, dispatch) { error: 'upload failed', [type]: id, }) + reject(evt) } function uploadCancelled (evt) { @@ -80,6 +83,7 @@ export function crud_upload(type, fd, data, dispatch) { error: 'upload cancelled', [type]: id, }) + reject(evt) } }) } diff --git a/app/client/common/fileList.component.js b/app/client/common/fileList.component.js index 7d90045..09e4268 100644 --- a/app/client/common/fileList.component.js +++ b/app/client/common/fileList.component.js @@ -12,9 +12,12 @@ class FileList extends Component { } render(){ const { files, linkFiles, onClick } = this.props + if (!files.length) return null let fields = this.props.fields || defaultFields const fileList = files.map(file => { const size = util.hush_size(file.size) + const date = file.created_at + const username = file.username || "" return (
@@ -23,11 +26,17 @@ class FileList extends Component { : onClick(file)}>{file.name || file.url} }
+ {fields.has('age') && +
{util.get_age(date)}
+ } + {fields.has('username') && +
{username}
+ } {fields.has('date') && -
{moment(file.created_at).format("YYYY-MM-DD")}
+
{moment(date).format("YYYY-MM-DD")}
} {fields.has('datetime') && -
{moment(file.created_at).format("YYYY-MM-DD h:mm a")}
+
{moment(date).format("YYYY-MM-DD h:mm a")}
} {fields.has('size') &&
{size[1]}
diff --git a/app/client/dashboard/dashboardHeader.component.js b/app/client/dashboard/dashboardHeader.component.js index 2e53088..701c97a 100644 --- a/app/client/dashboard/dashboardHeader.component.js +++ b/app/client/dashboard/dashboardHeader.component.js @@ -13,21 +13,45 @@ class DashboardHeader extends Component { this.props.onClick && this.props.onClick() } render() { - const { currentTask, site } = this.props - const eta = ((currentTask.epochs - currentTask.epoch) * 180 / 60) + " minutes" + const { site } = this.props return (

{site.name}

- Currently {util.gerund(currentTask.activity)} {currentTask.module} on {currentTask.dataset}
- Epoch: {currentTask.epoch} / {currentTask.epochs}, ETA {eta}
-
- Want to play live? + {this.renderGPUStatus()}
) } + renderGPUStatus(){ + const { runner } = this.props + const gpu = runner.cpu + if (gpu.status === 'IDLE') { + return null + } + const task = gpu.task + const eta = ((task.epochs - (task.epoch || 0)) * 180 / 60) + " minutes" + let activityPhrase, liveMessage + if (task.activity === 'live') { + return ( +
+ Currently running {task.module} live on "{task.dataset}" +
+ ) + } + else { + return ( +
+ Currently {util.gerund(task.activity)} {task.module} on {task.dataset}
+ Epoch: {task.epoch} / {task.epochs}, ETA {eta}
+
+ Want to play live? +
+ ) + } + } } const mapStateToProps = state => ({ + runner: state.system.runner, currentTask: state.task.currentTask, site: state.system.site, }) diff --git a/app/client/dataset/dataset.reducer.js b/app/client/dataset/dataset.reducer.js index c7a2e26..801f768 100644 --- a/app/client/dataset/dataset.reducer.js +++ b/app/client/dataset/dataset.reducer.js @@ -7,116 +7,36 @@ const datasetInitialState = { } const datasetReducer = (state = datasetInitialState, action) => { - console.log(action) switch(action.type) { - case types.socket.connect: - return { - ...state, - } - case types.task.task_begin: - return { - ...state, - } - case types.task.task_finish: - return { - ...state, - } - - case types.folder.index: - return { - ...state, - folders: action.data, - folder: action.data[0], - } - case types.folder.update: - return state - case types.file.index: - return { - ...state, - files: action.data - } - case types.folder.upload_loading: return { - ...state, - upload: { - loading: true, - status: 'Loading...', - }, + error: null, + loading: true, + status: 'Loading...', } case types.folder.upload_error: return { - ...state, - upload: { - loading: false, - status: 'Error uploading :(', - }, + error: null, + loading: false, + status: 'Error uploading :(', } case types.folder.upload_progress: - console.log(action) return { - ...state, - upload: { - loading: true, - status: 'Upload progress ' + action.percent + '%', - }, + error: null, + loading: true, + status: 'Upload progress ' + action.percent + '%', } case types.folder.upload_waiting: - console.log(action) return { - ...state, - upload: { - loading: true, - status: 'Waiting for server to finish processing...', - }, + error: null, + loading: true, + status: 'Waiting for server to finish processing...', } case types.file.create_loading: return { - ...state, - upload: { - loading: true, - status: 'Creating file...' - } - } - case types.file.create: - console.log('booo') - if (state.folder.id === action.data.folder_id) { - return { - ...state, - files: [action.data].concat(state.files), - upload: { - loading: false, - status: 'File created', - }, - } - } else { - return { - ...state, - upload: { - loading: false, - status: 'created', - }, - } - } - case types.folder.upload_complete: - console.log(action) - if (state.folder.id === action.folder) { - return { - ...state, - files: [action.files].concat(state.files), - upload: { - loading: false, - status: 'Upload complete', - }, - } - } else { - return { - ...state, - upload: { - loading: false, - status: 'Upload complete', - }, - } + error: null, + loading: true, + status: 'Creating file...' } case types.socket.status: return datasetSocket(state, action.data) diff --git a/app/client/modules/samplernn/samplernn.actions.js b/app/client/modules/samplernn/samplernn.actions.js index aa9603b..3386247 100644 --- a/app/client/modules/samplernn/samplernn.actions.js +++ b/app/client/modules/samplernn/samplernn.actions.js @@ -1,2 +1,42 @@ import socket from '../../socket' import types from '../../types' + +import actions from '../../actions' + +// bindActionCreators(actions.folder, dispatch), +// bindActionCreators(actions.file, dispatch), +// bindActionCreators(taskActions, dispatch), +// bindActionCreators(systemActions, dispatch), + +export const load_directories = () => (dispatch) => { + // load datasets + // load directories from server + console.log(actions) + actions.folder.index({ module: 'samplernn' }) + .then(folders => { + console.log('got folders') + }) + actions.file.index({ module: 'samplernn' }) + .then(files => { + console.log('got files') + }) + actions.socket.list_directory({ module: 'samplernn', dir: 'results' }) + .then(dirs => { + console.log('got results') + }) + actions.socket.list_directory({ module: 'samplernn', dir: 'datasets' }) + .then(dirs => { + console.log('got datasets') + }) +} + +export const fetch_url = (url) => (dispatch) => { + console.log(url) + actions.task.start_task({ + activity: 'fetch', + module: 'samplernn', + dataset: 'test', + epochs: 1, + opt: { url } + }, { preempt: true, watch: true }) +} diff --git a/app/client/modules/samplernn/samplernn.datasets.js b/app/client/modules/samplernn/samplernn.datasets.js index 5f15dbc..960c976 100644 --- a/app/client/modules/samplernn/samplernn.datasets.js +++ b/app/client/modules/samplernn/samplernn.datasets.js @@ -2,9 +2,7 @@ import { h, Component } from 'preact' import { bindActionCreators } from 'redux' import { connect } from 'react-redux' -import { actions, parser } from '../../api' -import * as taskActions from '../../task/task.actions' -import * as systemActions from '../../system/system.actions' +import * as samplernnActions from './samplernn.actions' import Dataset from '../../dataset/dataset.component' @@ -18,17 +16,15 @@ import TextInput from '../../common/textInput.component' class SampleRNNDatasets extends Component { constructor(props){ super() - // fetch file list this.fileOptions = this.fileOptions.bind(this) this.pickFile = this.pickFile.bind(this) - props.actions.folder.index({ module: 'samplernn' }) - props.actions.file.index({ module: 'samplernn' }) + props.actions.load_directories() } pickFile(file){ console.log('pick', file) } fileOptions(file){ - console.log(file) + // console.log(file) if (file.activity === 'url' && !file.dataset) { if (this.props.runner.cpu.status !== 'IDLE') { return ( @@ -52,18 +48,10 @@ class SampleRNNDatasets extends Component { ) } fetchURL(url) { - console.log(url) - this.props.actions.task.start_task({ - activity: 'fetch', - module: 'samplernn', - dataset: 'test', - epochs: 1, - opt: { url } - }, { preempt: true, watch: true }) } render(){ const { samplernn } = this.props - console.log(samplernn.upload) + // console.log(samplernn.upload) // sort files?? const module = { name: 'samplernn', @@ -98,12 +86,7 @@ const mapStateToProps = state => ({ }) const mapDispatchToProps = (dispatch, ownProps) => ({ - actions: { - folder: bindActionCreators(actions.folder, dispatch), - file: bindActionCreators(actions.file, dispatch), - task: bindActionCreators(taskActions, dispatch), - system: bindActionCreators(systemActions, dispatch), - } + actions: bindActionCreators(samplernnActions, dispatch), }) export default connect(mapStateToProps, mapDispatchToProps)(SampleRNNDatasets) diff --git a/app/client/modules/samplernn/samplernn.reducer.js b/app/client/modules/samplernn/samplernn.reducer.js index b3b58c3..9ecd492 100644 --- a/app/client/modules/samplernn/samplernn.reducer.js +++ b/app/client/modules/samplernn/samplernn.reducer.js @@ -3,6 +3,7 @@ import types from '../../types' const samplernnInitialState = { loading: false, error: null, + folder: {}, folders: [], datasets: [], results: [], @@ -17,8 +18,11 @@ const samplernnInitialState = { } const samplernnReducer = (state = samplernnInitialState, action) => { - console.log(action) switch(action.type) { + case types.socket.connect: + return { + ...state, + } case types.task.task_begin: return { ...state, @@ -27,7 +31,45 @@ const samplernnReducer = (state = samplernnInitialState, action) => { return { ...state, } + case types.folder.index: + return { + ...state, + folders: action.data, + folder: action.data[0], + } + case types.folder.update: + return state + + case types.file.index: + return { + ...state, + files: action.data + } + case types.file.create: + if (state.folder.id === action.data.folder_id) { + return { + ...state, + files: [action.data].concat(state.files), + } + } + return state + + case types.folder.upload_complete: + if (state.folder.id === action.folder) { + return { + ...state, + files: [action.files].concat(state.files), + } + } + return state + + case types.system.list_directory: + console.log('list directory', action.data) + return { + ...state, + } + case types.socket.status: return samplernnSocket(state, action.data) default: @@ -38,7 +80,9 @@ const samplernnReducer = (state = samplernnInitialState, action) => { const samplernnSocket = (state, action) => { console.log(action) switch (action.key) { - default: + case 'list_directory': + return state + default: return state } } diff --git a/app/client/socket/socket.actions.js b/app/client/socket/socket.actions.js new file mode 100644 index 0000000..574892a --- /dev/null +++ b/app/client/socket/socket.actions.js @@ -0,0 +1,3 @@ +import { list_directory_async } from './socket.system' + +export const list_directory = list_directory_async diff --git a/app/client/socket/socket.system.js b/app/client/socket/socket.system.js index a3c4f15..11cb44c 100644 --- a/app/client/socket/socket.system.js +++ b/app/client/socket/socket.system.js @@ -1,6 +1,6 @@ import { dispatch } from '../store' import types from '../types' - +import uuidv1 from 'uuid/v1' import { socket } from './socket.connection' socket.on('system_res', (data) => { @@ -28,11 +28,18 @@ socket.on('system_res', (data) => { type: data.rpc_connected ? types.system.rpc_connected : types.system.rpc_disconnected, runner: data.runner, }) - case 'command_output': - return dispatch({ - type: types.system.command_output, - data: data, - }) + // case 'run_system_command': + // return dispatch({ + // type: types.system.command_output, + // data: data, + // }) + // case 'list_directory': + // return dispatch({ + // type: types.system.list_directory, + // data: data, + // }) + // break + default: break } }) @@ -43,3 +50,33 @@ export function run_system_command(cmd) { payload: cmd, }) } + +export function list_directory(opt) { + socket.emit('system', { + cmd: 'list_directory', + payload: opt, + }) +} + +export function list_directory_async(opt) { + return syscall_async('list_directory', opt) +} + +export const syscall_async = (tag, payload, ttl=10000) => { + return new Promise( (resolve, reject) => { + const uuid = uuidv1() + const timeout = setTimeout(() => { + socket.off('system_res', cb) + reject('timeout') + }, ttl) + const cb = (data) => { + if (data.uuid === uuid) { + clearTimeout(timeout) + socket.off('system_res', cb) + resolve(data) + } + } + socket.emit('system', { cmd: tag, payload, uuid }) + socket.on('system_res', cb) + }) +} diff --git a/app/client/system/system.actions.js b/app/client/system/system.actions.js index 4c5f98e..a318d68 100644 --- a/app/client/system/system.actions.js +++ b/app/client/system/system.actions.js @@ -6,6 +6,11 @@ export const run = (cmd) => { return { type: types.system.running_command, cmd } } +export const listDirectory = (opt) => { + socket.system.list_directory(opt) + return { type: types.system.listing_directory, opt } +} + export const changeTool = (tool) => { return { type: types.app.change_tool, tool } } \ No newline at end of file diff --git a/app/client/types.js b/app/client/types.js index ad9613f..b6593ff 100644 --- a/app/client/types.js +++ b/app/client/types.js @@ -8,6 +8,8 @@ export default { relay_disconnected: 'SYSTEM_RELAY_DISCONNECTED', rpc_connected: 'SYSTEM_RPC_CONNECTED', rpc_disconnected: 'SYSTEM_RPC_DISCONNECTED', + list_directory: 'SYSTEM_LIST_DIRECTORY', + listing_directory: 'SYSTEM_LISTING_DIRECTORY', stdout: 'SYSTEM_STDOUT', stderr: 'SYSTEM_STDERR', }, diff --git a/app/client/util.js b/app/client/util.js index 04eb9c6..2ba6d3f 100644 --- a/app/client/util.js +++ b/app/client/util.js @@ -124,4 +124,25 @@ export function hush_null (n, unit, no_bold) { return ["new", s] } } +export function get_age (t) { + var age = Math.abs(+Date.now() - new Date(t))/1000 + var r = Math.floor + var m + if (age < 5) { return "now" } + if (age < 60) { return r(age) + "s" } + age /= 60 + if (age < 60) { return r(age) + "m" } + m = r(age % 60) + age /= 60 + if (m > 0 && age < 2) { return r(age) + "h" + m + "m" } + if (age < 24) { return r(age) + "h" } + age /= 24 + if (age < 7) { return r(age) + "d" } + age /= 7 + if (age < 12) { return r(age) + "w" } + age /= 4 + if (age < 12) { return r(age) + "m" } + age /= 12 + return r(age) + "y" +} export function courtesy_s (n, s) { return n == 1 ? "" : (s || "s") } diff --git a/app/relay/remote.js b/app/relay/remote.js index e468c7d..7c70c26 100644 --- a/app/relay/remote.js +++ b/app/relay/remote.js @@ -74,25 +74,41 @@ remote.on('task', (data) => { remote.on('system', (data) => { console.log('system:', data.cmd) + // console.log(data) switch(data.cmd) { case 'run_system_command': runner.run_system_command(data.payload, (error, stdout, stderr) => { remote.emit('system_res', { - type: 'command_output', + type: 'run_system_command', cmd: data.payload, + uuid: data.uuid, error, stdout, stderr }) }) break + case 'list_directory': + runner.list_directory(data.payload, (files) => { + remote.emit('system_res', { + type: 'list_directory', + dir: data.payload, + uuid: data.uuid, + files, + }) + }) case 'get_status': remote.emit('system_res', { type: 'relay_status', rpc_connected: get_connected(), + uuid: data.uuid, runner: runner.status(), }) break default: - remote.emit('system_res', { type: 'error', error: 'unknown system command' }) + remote.emit('system_res', { + type: 'error', + error: 'unknown system command', + uuid: data.uuid, + }) break } }) diff --git a/app/relay/runner.js b/app/relay/runner.js index 2713a2e..6620bca 100644 --- a/app/relay/runner.js +++ b/app/relay/runner.js @@ -5,6 +5,8 @@ import kill from 'tree-kill' import { remote } from './remote' import { set_connected } from './rpc' import uuidv1 from 'uuid/v1' +import * as fs from 'fs' +import * as path from 'path' const idle_state = { status: 'IDLE', task: {} } @@ -108,6 +110,33 @@ export function run_system_command(cmd, cb) { } } +export function list_directory(opt, cb) { + if (!opt.module || ! modules[opt.module]) { + cb([]) + } + const module = modules[opt.module] + const dir = path.join(module.cwd, opt.dir.replace(/\.\.?\//g, '')) + fs.readdir(dir, (err, files) => { + const statPromises = files.filter(f => f[0] !== '.').map(f => { + return new Promise((resolve, reject) => { + const full_path = path.join(dir, f) + fs.stat(full_path, (err, stat={}) => { + resolve({ + name: f, + path: full_path, + date: stat.ctime, + size: stat.size, + dir: stat.isDirectory(), + }) + }) + }) + }) + Promise.all(statPromises).then(stats => { + cb(stats) + }) + }) +} + export function run_task(task, preempt, watch){ const module = modules[task.module] if (! module) return { type: 'error', error: "No such module: " + task.module } @@ -136,6 +165,7 @@ export function run_task(task, preempt, watch){ console.log(module.cwd) console.log(interpreter.cmd, params) + task.started = new Date().toString() const subprocess = spawn(interpreter.cmd, params, { cwd: module.cwd, }) -- cgit v1.2.3-70-g09d2