disentangle dataset loader from samplernn

1 files changed, 164 insertions, 105 deletions

diff --git a/public/bundle.js b/public/bundle.js
index 0eb8310..84fe51b 100644
--- a/public/bundle.js
+++ b/public/bundle.js
@@ -3127,6 +3127,142 @@ exports.default = (0, _reactRedux.connect)(mapStateToProps, mapDispatchToProps)(
 
 /***/ }),
 
+/***/ "./app/client/dataset/dataset.loader.js":
+/*!**********************************************!*\
+  !*** ./app/client/dataset/dataset.loader.js ***!
+  \**********************************************/
+/*! no static exports found */
+/***/ (function(module, exports, __webpack_require__) {
+
+"use strict";
+
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.load = exports.getDataset = exports.emptyDataset = exports.unsortedFolder = undefined;
+
+var _slicedToArray = function () { function sliceIterator(arr, i) { var _arr = []; var _n = true; var _d = false; var _e = undefined; try { for (var _i = arr[Symbol.iterator](), _s; !(_n = (_s = _i.next()).done); _n = true) { _arr.push(_s.value); if (i && _arr.length === i) break; } } catch (err) { _d = true; _e = err; } finally { try { if (!_n && _i["return"]) _i["return"](); } finally { if (_d) throw _e; } } return _arr; } return function (arr, i) { if (Array.isArray(arr)) { return arr; } else if (Symbol.iterator in Object(arr)) { return sliceIterator(arr, i); } else { throw new TypeError("Invalid attempt to destructure non-iterable instance"); } }; }();
+
+var _actions = __webpack_require__(/*! ../actions */ "./app/client/actions.js");
+
+var _actions2 = _interopRequireDefault(_actions);
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
+var unsortedFolders = {};
+
+var unsortedFolder = exports.unsortedFolder = function unsortedFolder(module) {
+  var instantiate = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : false;
+
+  if (!unsortedFolders[module] || instantiate) {
+    var folder = {
+      id: 0,
+      module: module,
+      name: 'unsorted',
+      files: [],
+      datasets: []
+    };
+    unsortedFolders[module] = folder;
+  }
+  return unsortedFolders[module];
+};
+
+var emptyDataset = exports.emptyDataset = function emptyDataset(module, datasetLookup, name, folder) {
+  var dataset = {
+    name: name,
+    input: [],
+    checkpoints: [],
+    output: []
+  };
+  datasetLookup[name] = dataset;
+  folder = folder || unsortedFolder(module);
+  folder.datasets.push(name);
+  return dataset;
+};
+
+var getDataset = exports.getDataset = function getDataset(module, datasetLookup, name, folder, date) {
+  var dataset = datasetLookup[name] || emptyDataset(module, datasetLookup, name, folder);
+  folder = folder || unsortedFolder(module);
+  if (date) {
+    dataset.date = dataset.date && !isNaN(dataset.date) ? Math.max(+new Date(date), dataset.date) : +new Date(date);
+  }
+  return dataset;
+};
+
+var load = exports.load = function load(module) {
+  return Promise.all([_actions2.default.folder.index({ module: module }), _actions2.default.file.index({ module: module })]).then(function (res) {
+    var _res = _slicedToArray(res, 2),
+        folders = _res[0],
+        files = _res[1];
+
+    var datasetLookup = {};
+    var folderLookup = {};
+    var fileLookup = {};
+
+    // take all of the folders and put them in a lookup
+    folderLookup = folders.reduce(function (folderLookup, folder) {
+      folderLookup[folder.id] = {
+        id: folder.id,
+        name: folder.name,
+        folder: folder,
+        files: [],
+        datasets: []
+      };
+      return folderLookup;
+    }, {
+      unsorted: unsortedFolder(module, true)
+    });
+
+    // prepare the files by splitting into two groups
+    var generatedFiles = files.filter(function (file) {
+      return file.generated;
+    });
+    var ungeneratedFiles = files.filter(function (file) {
+      return !file.generated;
+    });
+
+    // build the initial dataset lookup table using the ungenerated files
+    ungeneratedFiles.reduce(function (datasetLookup, file) {
+      fileLookup[file.id] = file;
+      if (!file.name) {
+        file.name = (file.opt || {}).token || file.url;
+      }
+      var name = (file.name || 'unsorted').split('.')[0];
+      var folder = folderLookup[file.folder_id] || unsortedFolder(module);
+      var dataset = getDataset(module, datasetLookup, name, folder, file.date || file.created_at);
+      if (file.url.match(file.name)) file.persisted = true;
+      dataset.input.push(file.id);
+      folder.files.push(file.id);
+      return datasetLookup;
+    }, datasetLookup);
+
+    // go over the generated files and add addl datasets (if the files were deleted)
+    generatedFiles.map(function (file) {
+      fileLookup[file.id] = file;
+      var pair = file.name.split('.')[0].split('-');
+      var folder = folderLookup[file.folder_id] || unsortedFolder(module);
+      var dataset = getDataset(module, datasetLookup, pair[0], folder, file.date || file.created_at);
+      dataset.output.push(file.id);
+      folder.files.push(file.id);
+      file.epoch = file.epoch || pair[1];
+    });
+
+    return {
+      folderLookup: folderLookup,
+      fileLookup: fileLookup,
+      datasetLookup: datasetLookup,
+      folders: folders,
+      files: files,
+      unsortedFolder: folderLookup.unsorted
+    };
+  }).catch(function (e) {
+    console.error(e);
+  });
+};
+
+/***/ }),
+
 /***/ "./app/client/dataset/dataset.new.js":
 /*!*******************************************!*\
   !*** ./app/client/dataset/dataset.new.js ***!
@@ -5104,120 +5240,46 @@ var _types = __webpack_require__(/*! ../../types */ "./app/client/types.js");
 
 var _types2 = _interopRequireDefault(_types);
 
+var _dataset = __webpack_require__(/*! ../../dataset/dataset.loader */ "./app/client/dataset/dataset.loader.js");
+
+var datasetLoader = _interopRequireWildcard(_dataset);
+
 var _actions = __webpack_require__(/*! ../../actions */ "./app/client/actions.js");
 
 var _actions2 = _interopRequireDefault(_actions);
 
 var _util = __webpack_require__(/*! ../../util */ "./app/client/util/index.js");
 
+function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }
+
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
 function _toArray(arr) { return Array.isArray(arr) ? arr : Array.from(arr); }
 
 var load_directories = exports.load_directories = function load_directories(id) {
   return function (dispatch) {
-    // console.log(actions)
-    dispatch({ type: _types2.default.app.load_progress, progress: { i: 0, n: 7 } });
-    (0, _util.allProgress)([_actions2.default.folder.index({ module: 'samplernn' }), _actions2.default.file.index({ module: 'samplernn' }), _actions2.default.task.index({ module: 'samplernn' }), _actions2.default.socket.list_directory({ module: 'samplernn', dir: 'datasets' }), _actions2.default.socket.list_directory({ module: 'samplernn', dir: 'results' }), _actions2.default.socket.list_directory({ module: 'samplernn', dir: 'output' }), _actions2.default.socket.disk_usage({ module: 'samplernn', dir: 'datasets' }), load_loss()(dispatch)], function (percent, i, n) {
+    var module = 'samplernn';
+    (0, _util.allProgress)([datasetLoader.load(module), _actions2.default.task.index({ module: module }), _actions2.default.socket.list_directory({ module: module, dir: 'datasets' }), _actions2.default.socket.list_directory({ module: module, dir: 'results' }), _actions2.default.socket.list_directory({ module: module, dir: 'output' }), _actions2.default.socket.disk_usage({ module: module, dir: 'datasets' }), load_loss()(dispatch)], function (percent, i, n) {
       dispatch({ type: _types2.default.app.load_progress, progress: { i: i, n: n } });
     }).then(function (res) {
       // console.log(res)
-      var _res = _slicedToArray(res, 8),
-          folders = _res[0],
-          files = _res[1],
-          tasks = _res[2],
-          datasets = _res[3],
-          results = _res[4],
-          output = _res[5],
-          datasetUsage = _res[6],
-          lossReport = _res[7];
-
-      console.log(datasetUsage);
-
-      var unsortedFolder = {
-        id: 0,
-        name: 'unsorted',
-        files: [],
-        datasets: []
-      };
-
-      var datasetLookup = {};
-      var folderLookup = {};
-      var fileLookup = {};
-      var taskLookup = {};
-
-      var get_dataset = function get_dataset(name) {
-        var folder = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : unsortedFolder;
-        var date = arguments[2];
+      var _res = _slicedToArray(res, 7),
+          datasetApiReport = _res[0],
+          tasks = _res[1],
+          datasets = _res[2],
+          results = _res[3],
+          output = _res[4],
+          datasetUsage = _res[5],
+          lossReport = _res[6];
 
-        var dataset = datasetLookup[name] || empty_dataset(name, folder);
-        if (date) {
-          dataset.date = dataset.date && !isNaN(dataset.date) ? Math.max(+new Date(date), dataset.date) : +new Date(date);
-        }
-        return dataset;
-      };
-
-      var empty_dataset = function empty_dataset(name) {
-        var folder = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : unsortedFolder;
-
-        var dataset = {
-          name: name,
-          input: [],
-          checkpoints: [],
-          output: []
-        };
-        datasetLookup[name] = dataset;
-        folder.datasets.push(name);
-        return dataset;
-      };
-
-      // take all of the folders and put them in a lookup
-      folderLookup = folders.reduce(function (folderLookup, folder) {
-        folderLookup[folder.id] = {
-          id: folder.id,
-          name: folder.name,
-          folder: folder,
-          files: [],
-          datasets: []
-        };
-        return folderLookup;
-      }, {
-        unsorted: unsortedFolder
-      });
+      var folderLookup = datasetApiReport.folderLookup,
+          fileLookup = datasetApiReport.fileLookup,
+          datasetLookup = datasetApiReport.datasetLookup,
+          folders = datasetApiReport.folders,
+          files = datasetApiReport.files,
+          unsortedFolder = datasetApiReport.unsortedFolder;
 
-      // prepare the files by splitting into two groups
-      var generatedFiles = files.filter(function (file) {
-        return file.generated;
-      });
-      var ungeneratedFiles = files.filter(function (file) {
-        return !file.generated;
-      });
-
-      // build the initial dataset lookup table using the ungenerated files
-      ungeneratedFiles.reduce(function (datasetLookup, file) {
-        fileLookup[file.id] = file;
-        if (!file.name) {
-          file.name = (file.opt || {}).token || file.url;
-        }
-        var name = (file.name || 'unsorted').split('.')[0];
-        var folder = folderLookup[file.folder_id] || unsortedFolder;
-        var dataset = get_dataset(name, folder, file.date || file.created_at);
-        if (file.url.match(file.name)) file.persisted = true;
-        dataset.input.push(file.id);
-        folder.files.push(file.id);
-        return datasetLookup;
-      }, datasetLookup);
-
-      // go over the generated files and add addl datasets (if the files were deleted)
-      generatedFiles.map(function (file) {
-        fileLookup[file.id] = file;
-        var pair = file.name.split('.')[0].split('-');
-        var folder = folderLookup[file.folder_id] || unsortedFolder;
-        var dataset = get_dataset(pair[0], folder, file.date || file.created_at);
-        dataset.output.push(file.id);
-        folder.files.push(file.id);
-        file.epoch = file.epoch || pair[1];
-      });
+      console.log(datasetUsage);
 
       // also show the various flat audio files we have, in the input area..
       var flatDatasets = datasets.filter(function (s) {
@@ -5227,7 +5289,7 @@ var load_directories = exports.load_directories = function load_directories(id)
         return s.dir;
       });
       builtDatasets.forEach(function (dir) {
-        var dataset = get_dataset(dir.name);
+        var dataset = datasetLoader.getDataset(module, datasetLookup, dir.name);
         dataset.isBuilt = true;
       });
 
@@ -5235,7 +5297,7 @@ var load_directories = exports.load_directories = function load_directories(id)
         file.uuid = (0, _v2.default)();
         fileLookup[file.uuid] = file;
         var name = file.name.split('.')[0];
-        var dataset = get_dataset(name, unsortedFolder, file.date);
+        var dataset = datasetLoader.getDataset(module, datasetLookup, name, unsortedFolder, file.date);
         file.persisted = false;
         dataset.input.push(file.uuid);
       });
@@ -5255,7 +5317,7 @@ var load_directories = exports.load_directories = function load_directories(id)
         checkpoint.date = s.date;
         checkpoint.dir = s;
         checkpoint.persisted = false;
-        var dataset = get_dataset(checkpoint.name, unsortedFolder, checkpoint.date);
+        var dataset = datasetLoader.getDataset(module, datasetLookup, checkpoint.name, unsortedFolder, checkpoint.date);
         var loss = lossReport[checkpoint.name];
         if (loss) {
           dataset.epoch = checkpoint.epoch = loss.length;
@@ -5269,7 +5331,7 @@ var load_directories = exports.load_directories = function load_directories(id)
         file.uuid = (0, _v2.default)();
         fileLookup[file.uuid] = file;
         var pair = file.name.split('.')[0].split('-');
-        var dataset = get_dataset(pair[0], unsortedFolder, file.date);
+        var dataset = datasetLoader.getDataset(module, datasetLookup, pair[0], unsortedFolder, file.date);
         file.persisted = false;
         file.epoch = parseInt(file.epoch || pair[1].replace(/^\D+/, '')) || 0;
         dataset.epoch = Math.max(file.epoch, dataset.epoch || 0);
@@ -5293,7 +5355,6 @@ var load_directories = exports.load_directories = function load_directories(id)
           folderLookup: folderLookup,
           fileLookup: fileLookup,
           datasetLookup: datasetLookup,
-          taskLookup: taskLookup,
           folders: folders, files: files,
           checkpoints: checkpoints,
           output: output
@@ -5677,9 +5738,7 @@ var mapStateToProps = function mapStateToProps(state) {
 };
 
 var mapDispatchToProps = function mapDispatchToProps(dispatch, ownProps) {
-  return {
-    samplernnActions: (0, _redux.bindActionCreators)(samplernnActions, dispatch)
-  };
+  return {};
 };
 
 exports.default = (0, _reactRedux.connect)(mapStateToProps, mapDispatchToProps)(SampleRNNDatasets);