fix p2x fetch script

4 files changed, 174 insertions, 163 deletions

diff --git a/app/client/dataset/dataset.actions.js b/app/client/dataset/dataset.actions.js
index ed17418..1d28910 100644
--- a/app/client/dataset/dataset.actions.js
+++ b/app/client/dataset/dataset.actions.js
@@ -62,6 +62,11 @@ export const fetchURL = (module, folder, url) => dispatch => {
     console.log('media', media)
     actions.file.create({
       folder_id: folder.id,
+      name: (media.title || "")
+              .replace(/\.[^.]+$/g, '')
+              .replace(/[^a-zA-Z0-9_]/g, '_')
+              .replace(/\s+/g, '_')
+              .replace(/_+/g, '_'),
       module: module.name,
       dataset: media.token,
       activity: 'url',
diff --git a/app/client/dataset/dataset.reducer.js b/app/client/dataset/dataset.reducer.js
index d7951d0..6b51715 100644
--- a/app/client/dataset/dataset.reducer.js
+++ b/app/client/dataset/dataset.reducer.js
@@ -35,181 +35,184 @@ const datasetReducer = (state = datasetInitialState(), action) => {
       }
 
     case types.folder.create:
-      if (action.data.module === 'samplernn') {
-        return {
-          ...state,
-          loading: false,
-          data: {
-            folders: state.data.folders.concat([action.data.id]),
-            folderLookup: {
-              ...state.data.folderLookup,
-              [action.data.id]: action.data,
-            }
-          },
-        }
+      return {
+        ...state,
+        loading: false,
+        data: {
+          folders: state.data.folders.concat([action.data.id]),
+          folderLookup: {
+            ...state.data.folderLookup,
+            [action.data.id]: action.data,
+          }
+        },
       }
       return state
 
     case types.file.create:
-      if (action.data.module === 'samplernn') {
-        console.log(action.data)
-        let dataset, old_dataset, folder, old_folder
-        let dataset_name
-        if (action.data.dataset) {
-          dataset_name = action.data.dataset
-        }
-        else if (action.data.name) {
-          dataset_name = action.data.name.split('.')[0]
-        }
-        else if (action.data.url) {
-          dataset_name = action.data.url
-        }
-        else {
-          dataset_name = null
-        }
-        if (dataset_name in state.data.datasetLookup) {
-          old_dataset = state.data.datasetLookup[dataset_name]
-          dataset = {
-            ...old_dataset,
-            input: action.data.generated ? old_dataset.input : [action.data.id].concat(old_dataset.input),
-            output: !action.data.generated ? old_dataset.output : [action.data.id].concat(old_dataset.output),
-          }
-        } else {
-          console.log(action.data.created_at, action.data.date)
-          dataset = {
-            name: dataset_name,
-            date: action.data.created_at || action.data.date,
-            input: action.data.generated ? [] : [action.data.id],
-            checkpoints: [],
-            output: !action.data.generated ? [] : [action.data.id],
-          }
-        }
-        old_folder = state.data.folderLookup[action.data.folder_id]
-        folder = {
-          ...old_folder,
-          datasets: old_folder.datasets.concat([dataset_name]),
-          files: old_folder.files.concat([action.data.id]),
-        }
-        return {
-          ...state,
-          loading: false,
-          data: {
-            ...state.data,
-            files: state.data.files.concat([action.data.id]),
-            folderLookup: {
-              ...state.data.folderLookup,
-              [action.data.folder_id]: folder,
-            },
-            fileLookup: {
-              ...state.data.fileLookup,
-              [action.data.id]: action.data,
-            },
-            datasetLookup: {
-              ...state.data.datasetLookup,
-              [dataset_name]: dataset,
-            },
-          }
-        }
-      }
-      return state
+      return handleFileCreate(state, action)
 
     case types.file.update:
-      if (action.data.module === 'samplernn') {
-        let old_dataset;
-        let new_dataset = state.data.datasetLookup[action.data.dataset]
-        let old_file = state.data.fileLookup[action.data.id]
-        let new_dataset_update;
-        if (old_file && action.data.dataset !== old_file.dataset) {
-          if (state.data.datasetLookup[old_file.dataset]) {
-            old_dataset = state.data.datasetLookup[old_file.dataset]
-            old_dataset_update = {
-              ...old_dataset,
-              input: old_dataset.input.filter(id => id !== action.data.id),
-              output: old_dataset.output.filter(id => id !== action.data.id),
-            }
-          }
-          new_dataset_update = {
-            ...new_dataset,
-            input: action.data.generated ? [] : [action.data.id],
-            output: !action.data.generated ? [] : [action.data.id],
-          }
-        }
-        console.log('old_dataset', old_dataset_update.name)
-        console.log('new_dataset', new_dataset_update.name)
-        return {
-          ...state,
-          loading: false,
-          data: {
-            ...state.data,
-            datasetLookup: old_dataset ? {
-              ...state.data.datasetLookup,
-              [old_dataset.name]: old_dataset_update,
-              [new_dataset.name]: new_dataset_update,
-            } : new_dataset_update ? {
-              ...state.data.datasetLookup,
-              [new_dataset.name]: new_dataset_update,
-            } : state.data.datasetLookup,
-            fileLookup: {
-              ...state.data.fileLookup,
-              [action.data.id]: action.data,
-            },  
-          }
-        }
-      }
-      return state
+      return handleFileUpdate(state, action)
 
     case types.file.destroy:
-      if (action.data.module === 'samplernn') {
-        // delete the key from fileLookup
-        let fileLookup = {
-          ...state.data.fileLookup
-        }
-        let datasetLookup = {
-          ...state.data.datasetLookup
-        }
-        let folderLookup = {
-          ...state.data.folderLookup
-        }
-        let old_file = fileLookup[action.data.id]
-        if (!old_file) return state
-        let old_dataset = state.data.datasetLookup[old_file.dataset]
-        let dataset
-        if (old_dataset) {
-          dataset = {
-            ...old_dataset,
-            input: old_dataset.input.filter(id => id !== old_file.id),
-            output: old_dataset.output.filter(id => id !== old_file.id),
-          }
-          if (! dataset.input.length && ! dataset.checkpoints.length && ! dataset.output.length) {
-            delete datasetLookup[old_file.dataset]
-            let old_folder = folderLookup[old_file.folder_id]
-            folderLookup[old_file.folder_id] = {
-              ...old_folder,
-              datasets: old_folder.datasets.filter(name => name !== old_file.dataset)
-            }
-          } else {
-            console.log('dataset just needs sweeping')
-            datasetLookup[old_file.dataset] = dataset
-          }
-        }
-        delete fileLookup[old_file.id]
-
-        return {
-          ...state,
-          loading: false,
-          data: {
-            ...state.data,
-            folderLookup,
-            fileLookup,
-            datasetLookup,
-          }
-        }
-      }
-      return state
+      return handleFileDestroy(state, action)
 
     default:
       return state
   }
 }
 
+
+function handleFileCreate(state, action){
+  console.log(action.data)
+  let dataset, old_dataset, folder, old_folder
+  let dataset_name
+  if (action.data.dataset) {
+    dataset_name = action.data.dataset
+  }
+  else if (action.data.name) {
+    dataset_name = action.data.name.split('.')[0]
+  }
+  else if (action.data.url) {
+    dataset_name = action.data.url
+  }
+  else {
+    dataset_name = null
+  }
+  if (dataset_name in state.data.datasetLookup) {
+    old_dataset = state.data.datasetLookup[dataset_name]
+    dataset = {
+      ...old_dataset,
+      input: action.data.generated ? old_dataset.input : [action.data.id].concat(old_dataset.input),
+      output: !action.data.generated ? old_dataset.output : [action.data.id].concat(old_dataset.output),
+    }
+  } else {
+    console.log(action.data.created_at, action.data.date)
+    dataset = {
+      name: dataset_name,
+      date: action.data.created_at || action.data.date,
+      input: action.data.generated ? [] : [action.data.id],
+      checkpoints: [],
+      output: !action.data.generated ? [] : [action.data.id],
+    }
+  }
+  old_folder = state.data.folderLookup[action.data.folder_id]
+  folder = {
+    ...old_folder,
+    datasets: old_folder.datasets.concat([dataset_name]),
+    files: old_folder.files.concat([action.data.id]),
+  }
+  return {
+    ...state,
+    loading: false,
+    data: {
+      ...state.data,
+      files: state.data.files.concat([action.data.id]),
+      folderLookup: {
+        ...state.data.folderLookup,
+        [action.data.folder_id]: folder,
+      },
+      fileLookup: {
+        ...state.data.fileLookup,
+        [action.data.id]: action.data,
+      },
+      datasetLookup: {
+        ...state.data.datasetLookup,
+        [dataset_name]: dataset,
+      },
+    }
+  }
+}
+
+function handleFileUpdate(state, action){
+  let old_dataset;
+  let new_dataset = state.data.datasetLookup[action.data.dataset]
+  let old_file = state.data.fileLookup[action.data.id]
+  let new_dataset_update;
+  if (old_file && action.data.dataset !== old_file.dataset) {
+    if (state.data.datasetLookup[old_file.dataset]) {
+      old_dataset = state.data.datasetLookup[old_file.dataset]
+      old_dataset_update = {
+        ...old_dataset,
+        input: old_dataset.input.filter(id => id !== action.data.id),
+        output: old_dataset.output.filter(id => id !== action.data.id),
+      }
+    }
+    new_dataset_update = {
+      ...new_dataset,
+      input: action.data.generated ? [] : [action.data.id],
+      output: !action.data.generated ? [] : [action.data.id],
+    }
+  }
+  console.log('old_dataset', old_dataset_update.name)
+  console.log('new_dataset', new_dataset_update.name)
+  return {
+    ...state,
+    loading: false,
+    data: {
+      ...state.data,
+      datasetLookup: old_dataset ? {
+        ...state.data.datasetLookup,
+        [old_dataset.name]: old_dataset_update,
+        [new_dataset.name]: new_dataset_update,
+      } : new_dataset_update ? {
+        ...state.data.datasetLookup,
+        [new_dataset.name]: new_dataset_update,
+      } : state.data.datasetLookup,
+      fileLookup: {
+        ...state.data.fileLookup,
+        [action.data.id]: action.data,
+      },  
+    }
+  }
+  return state
+}
+
+function handleFileDestroy(state, action){
+  // delete the key from fileLookup
+  let fileLookup = {
+    ...state.data.fileLookup
+  }
+  let datasetLookup = {
+    ...state.data.datasetLookup
+  }
+  let folderLookup = {
+    ...state.data.folderLookup
+  }
+  let old_file = fileLookup[action.data.id]
+  if (!old_file) return state
+  let old_dataset = state.data.datasetLookup[old_file.dataset]
+  let dataset
+  if (old_dataset) {
+    dataset = {
+      ...old_dataset,
+      input: old_dataset.input.filter(id => id !== old_file.id),
+      output: old_dataset.output.filter(id => id !== old_file.id),
+    }
+    if (! dataset.input.length && ! dataset.checkpoints.length && ! dataset.output.length) {
+      delete datasetLookup[old_file.dataset]
+      let old_folder = folderLookup[old_file.folder_id]
+      folderLookup[old_file.folder_id] = {
+        ...old_folder,
+        datasets: old_folder.datasets.filter(name => name !== old_file.dataset)
+      }
+    } else {
+      console.log('dataset just needs sweeping')
+      datasetLookup[old_file.dataset] = dataset
+    }
+  }
+  delete fileLookup[old_file.id]
+
+  return {
+    ...state,
+    loading: false,
+    data: {
+      ...state.data,
+      folderLookup,
+      fileLookup,
+      datasetLookup,
+    }
+  }
+}
+
 export default datasetReducer
diff --git a/app/client/modules/pix2pix/pix2pix.actions.js b/app/client/modules/pix2pix/pix2pix.actions.js
index e15ea86..b5da04c 100644
--- a/app/client/modules/pix2pix/pix2pix.actions.js
+++ b/app/client/modules/pix2pix/pix2pix.actions.js
@@ -35,9 +35,11 @@ export const load_directories = (id) => (dispatch) => {
     // console.log(datasetUsage)
 
     const sequenceDirectories = sequences.filter(s => s.dir)
+    console.log(sequenceDirectories)
     sequenceDirectories.forEach(dir => {
       const dataset = datasetLoader.getDataset(module, datasetLookup, dir.name)
       dataset.isBuilt = true
+      console.log(dir.name, dataset)
     })
 
     datasets.filter(s => s.dir).forEach(dir => {
diff --git a/app/client/modules/pix2pix/views/pix2pix.show.js b/app/client/modules/pix2pix/views/pix2pix.show.js
index 9c55b0b..2713559 100644
--- a/app/client/modules/pix2pix/views/pix2pix.show.js
+++ b/app/client/modules/pix2pix/views/pix2pix.show.js
@@ -79,6 +79,7 @@ class Pix2pixShow extends Component {
     const input = pix2pix.data.fileLookup[dataset.input[0]]
     if (! input) return null
     if (input.name && input.name.match(/(gif|jpe?g|png)$/i)) return null
+    console.log(dataset)
     return (
       <div>
         <div class={'actions'}>