disentangle dataset loader from samplernn

1 files changed, 108 insertions, 0 deletions

diff --git a/app/client/dataset/dataset.loader.js b/app/client/dataset/dataset.loader.js
new file mode 100644
index 0000000..6757593
--- /dev/null
+++ b/app/client/dataset/dataset.loader.js
@@ -0,0 +1,108 @@
+
+import actions from '../actions'
+
+const unsortedFolders = {}
+
+export const unsortedFolder = (module, instantiate=false) => {
+  if (!unsortedFolders[module] || instantiate) {
+    const folder = {
+      id: 0,
+      module,
+      name: 'unsorted',
+      files: [],
+      datasets: [],
+    }
+    unsortedFolders[module] = folder
+  }
+  return unsortedFolders[module]
+}
+
+export const emptyDataset = (module, datasetLookup, name, folder) => {
+  const dataset = {
+    name,
+    input: [],
+    checkpoints: [],
+    output: [],
+  }
+  datasetLookup[name] = dataset
+  folder = folder || unsortedFolder(module)
+  folder.datasets.push(name)
+  return dataset
+}
+
+export const getDataset = (module, datasetLookup, name, folder, date) => {
+  const dataset = datasetLookup[name] || emptyDataset(module, datasetLookup, name, folder)
+  folder = folder || unsortedFolder(module)
+  if (date) {
+    dataset.date = (dataset.date && ! isNaN(dataset.date)) ? Math.max(+new Date(date), dataset.date) : +new Date(date)
+  }
+  return dataset
+}
+
+export const load = module => {
+  return Promise.all([
+    actions.folder.index({ module }),
+    actions.file.index({ module }),
+  ]).then(res => {
+    const [folders, files] = res
+
+    let datasetLookup = {}
+    let folderLookup = {}
+    let fileLookup = {}
+
+    // take all of the folders and put them in a lookup
+    folderLookup = folders.reduce((folderLookup, folder) => {
+      folderLookup[folder.id] = {
+        id: folder.id,
+        name: folder.name,
+        folder,
+        files: [],
+        datasets: [],
+      }
+      return folderLookup
+    }, {
+      unsorted: unsortedFolder(module, true)
+    })
+
+    // prepare the files by splitting into two groups
+    const generatedFiles = files.filter(file => file.generated)
+    const ungeneratedFiles = files.filter(file => !file.generated)
+
+    // build the initial dataset lookup table using the ungenerated files
+    ungeneratedFiles.reduce((datasetLookup, file) => {
+      fileLookup[file.id] = file
+      if (! file.name) {
+        file.name = (file.opt || {}).token || file.url
+      }
+      const name = (file.name || 'unsorted').split('.')[0]
+      const folder = folderLookup[file.folder_id] || unsortedFolder(module)
+      const dataset = getDataset(module, datasetLookup, name, folder, file.date || file.created_at)
+      if (file.url.match(file.name)) file.persisted = true
+      dataset.input.push(file.id)
+      folder.files.push(file.id)
+      return datasetLookup
+    }, datasetLookup)
+
+    // go over the generated files and add addl datasets (if the files were deleted)
+    generatedFiles.map(file => {
+      fileLookup[file.id] = file
+      const pair = file.name.split('.')[0].split('-')
+      const folder = folderLookup[file.folder_id] || unsortedFolder(module)
+      const dataset = getDataset(module, datasetLookup, pair[0], folder, file.date || file.created_at)
+      dataset.output.push(file.id)
+      folder.files.push(file.id)
+      file.epoch = file.epoch || pair[1]
+    })
+
+    return {
+      folderLookup,
+      fileLookup,
+      datasetLookup,
+      folders,
+      files,
+      unsortedFolder: folderLookup.unsorted,
+    }
+  }).catch(e => {
+    console.error(e)
+  })
+}