import actions from '../actions' const unsortedFolders = {} export const unsortedFolder = (module, instantiate=false) => { if (!unsortedFolders[module] || instantiate) { const folder = { id: 0, module, name: 'unsorted', files: [], datasets: [], } unsortedFolders[module] = folder } return unsortedFolders[module] } export const emptyDataset = (module, datasetLookup, name, folder) => { const dataset = { name, input: [], checkpoints: [], output: [], } datasetLookup[name] = dataset folder = folder || unsortedFolder(module) folder.datasets.push(name) return dataset } export const getDataset = (module, datasetLookup, name, folder, date) => { let dataset = datasetLookup[name] || emptyDataset(module, datasetLookup, name, folder) // folder = folder || unsortedFolder(module) if (date) { dataset.date = (dataset.date && ! isNaN(dataset.date)) ? Math.max(+new Date(date), dataset.date) : +new Date(date) } return dataset } export const load = module => { console.log(module) return Promise.all([ actions.folder.index({ module, limit: 0 }), actions.file.index({ module, limit: 0 }), actions.task.index({ module }), ]).then(res => { const [folders, files, tasks] = res let datasetLookup = {} let folderLookup = {} let fileLookup = {} // take all of the folders and put them in a lookup folderLookup = folders.reduce((folderLookup, folder) => { folderLookup[folder.id] = { id: folder.id, name: folder.name, folder, files: [], datasets: [], } return folderLookup }, { unsorted: unsortedFolder(module, true) }) // prepare the files by splitting into two groups const generatedFiles = files.filter(file => file.generated) const ungeneratedFiles = files.filter(file => !file.generated) // build the initial dataset lookup table using the ungenerated files ungeneratedFiles.reduce((datasetLookup, file) => { fileLookup[file.id] = file if (! file.name) { if (file.opt && file.opt.media && file.opt.media.token) { file.name = file.opt.media.token } else { file.name = file.url.replace(/^https?:\/\/(www.)?/,'').replace(/[^a-zA-Z0-9_]/g, '_').replace(/_+/g, '_') || 'unknown_' + Math.floor(1000*Math.random()) } } const name = (file.name || 'unsorted').split('.')[0] const folder = folderLookup[file.folder_id] || unsortedFolder(module) const dataset = getDataset(module, datasetLookup, file.dataset || name, folder, file.date || file.created_at) if (file.url.match(file.name)) file.persisted = true dataset.input.push(file.id) folder.files.push(file.id) return datasetLookup }, datasetLookup) // go over the generated files and add addl datasets (if the files were deleted) generatedFiles.map(file => { fileLookup[file.id] = file const pair = file.name.split('.')[0].split('-') const folder = folderLookup[file.folder_id] || unsortedFolder(module) const dataset = getDataset(module, datasetLookup, file.dataset || pair[0], folder, file.date || file.created_at) dataset.output.push(file.id) folder.files.push(file.id) file.epoch = file.epoch || pair[1] }) return { folderLookup, fileLookup, datasetLookup, folders, files, unsortedFolder: folderLookup.unsorted, } }).catch(e => { console.error(e) }) }