parse urls using the old parser

4 files changed, 390 insertions, 26 deletions

diff --git a/app/client/api/index.js b/app/client/api/index.js
index a0cae50..2fcd434 100644
--- a/app/client/api/index.js
+++ b/app/client/api/index.js
@@ -1,5 +1,6 @@
 import { crud_actions } from './crud.actions'
 import * as util from './util'
+import * as parser from './parser'
 
 /*
 for our crud events, create corresponding actions
@@ -14,6 +15,8 @@ so you can do ...
   folderActions.upload(12, form_data)
 */
 
+export { util, parser }
+
 export const actions = [
   'folder',
   'file',
diff --git a/app/client/api/parser.js b/app/client/api/parser.js
new file mode 100644
index 0000000..2530410
--- /dev/null
+++ b/app/client/api/parser.js
@@ -0,0 +1,329 @@
+import fetch from 'node-fetch'
+import fetchJsonp from 'fetch-jsonp'
+
+export const integrations = [
+  {
+    type: 'image',
+    regex: /\.(jpeg|jpg|gif|png|svg)(\?.*)?$/i,
+    fetch: function(url, done) {
+      var img = new Image ()
+      img.onload = function(){
+        if (!img) return
+        var width = img.naturalWidth, height = img.naturalHeight
+        img = null
+        done({
+          url: url,
+          type: "image",
+          token: "",
+          thumbnail: "",
+          title: "",
+          width: width,
+          height: height,
+        })
+      }
+      img.src = url
+      if (img.complete) {
+        img.onload()
+      }
+    },
+    tag: function (media) {
+      return '<img src="' + media.url + '">';
+    }
+  }, {
+    type: 'video',
+    regex: /\.(mp4|webm)(\?.*)?$/i,
+    fetch: function(url, done) {
+      var video = document.createElement("video")
+      var url_parts = url.replace(/\?.*$/, "").split("/")
+      var filename = url_parts[ url_parts.length-1 ]
+      video.addEventListener("loadedmetadata", function(){
+        var width = video.videoWidth, height = video.videoHeight
+        video = null
+        done({
+          url: url,
+          type: "video",
+          token: url,
+          thumbnail: "/public/assets/img/video-thumbnail.png",
+          title: filename,
+          width: width,
+          height: height,
+        })
+      })
+      video.src = url
+      video.load()
+    },
+    tag: function (media) {
+      return '<video src="' + media.url + '">';
+    }
+  }, {
+    type: 'audio',
+    regex: /\.(wav|mp3)(\?.*)?$/i,
+    fetch: function(url, done) {
+      var audio = document.createElement("audio")
+      var url_parts = url.replace(/\?.*$/, "").split("/")
+      var filename = url_parts[ url_parts.length-1 ]
+      audio.addEventListener("loadedmetadata", function(){
+        var duration = audio.duration
+        audio = null
+        done({
+          url: url,
+          type: "audio",
+          token: url,
+          thumbnail: "/public/assets/img/audio-thumbnail.png",
+          title: filename,
+          duration: duration,
+        })
+      })
+      audio.src = url
+      audio.load()
+    },
+    tag: function (media) {
+      return '<audio src="' + media.url + '">';
+    }
+  }, {
+    type: 'youtube',
+    regex:  /(?:youtube\.com\/(?:[^\/]+\/.+\/|(?:v|e(?:mbed)?)\/|.*[?&]v=)|youtu\.be\/)([^"&?\/ ]{11})/i,
+    fetch: function(url, done) {
+      var id = (url.match(/v=([-_a-zA-Z0-9]{11})/i) || url.match(/youtu.be\/([-_a-zA-Z0-9]{11})/i) || url.match(/embed\/([-_a-zA-Z0-9]{11})/i))[1].split('&')[0];
+      var thumb = "https://i.ytimg.com/vi/" + id + "/hqdefault.jpg"
+
+      const jsonp_url = new URL('https://www.googleapis.com/youtube/v3/videos')
+      if (data) {
+        Object.keys({
+          id: id,
+          key: "AIzaSyCaLRGY-hxs92045X-Jew7w1FgQPkStHgc",
+          part: "id,contentDetails,snippet,status",
+        }).forEach(key => jsonp_url.searchParams.append(key, data[key]))
+      }
+
+      fetchJsonp(jsonp_url)
+      .then(result => result.json())
+      .then(result => {
+        if (! result || ! result.items.length) {
+          return alert("Sorry, this video URL is invalid.")
+        }
+        var res = result.items[0]
+        // console.log(res)
+        
+        var dd = res.contentDetails.duration.match(/\d+/g).map(function(i){ return parseInt(i) })
+        var duration = 0
+        if (dd.length == 3) {
+          duration = ((dd[0] * 60) + dd[1] * 60) + dd[2]
+        }
+        else if (dd.length == 2) {
+          duration = (dd[0] * 60) + dd[1]
+        }
+        else {
+          duration = dd[0]
+        }
+        
+        ["maxres","high","medium","standard","default"].some(function(t){
+          if (res.snippet.thumbnails[t]) {
+            thumb = res.snippet.thumbnails[t].url
+            return true
+          }
+          return false
+        })
+        
+        done({
+          url: url,
+          type: "youtube",
+          token: id,
+          thumbnail: thumb,
+          title: res.snippet.title,
+          duration: duration,
+          width: 640,
+          height: 360,
+        })
+      })
+    },
+    tag: function (media) {
+      // return '<img class="video" type="youtube" vid="'+media.token+'" src="'+media.thumbnail+'"><span class="playvid">&#9654;</span>';
+      return '<div class="video" style="width: ' + media.width + 'px; height: ' + media.height + 'px; overflow: hidden; position: relative;"><iframe frameborder="0" scrolling="no" seamless="seamless" webkitallowfullscreen="webkitAllowFullScreen" mozallowfullscreen="mozallowfullscreen" allowfullscreen="allowfullscreen" id="okplayer" width="' + media.width + '" height="' + media.height + '" src="https://youtube.com/embed/' + media.token + '?showinfo=0" style="position: absolute; top: 0px; left: 0px; width: ' + media.width + 'px; height: ' + media.height + 'px;"></iframe></div>'
+    }
+  }, {
+    type: 'vimeo',
+    regex: /vimeo.com\/\d+$/i,
+    fetch: function(url, done) {
+      var id = url.match(/\d+$/i)[0];
+      fetch('https://vimeo.com/api/v2/video/' + id + '.json')
+      .then(result => result.json())
+      .then(result => {
+        if (result.length == 0) { return done(id, "", 640, 360) }
+        var res = result[0]
+        if (res.embed_privacy != "anywhere") {
+          alert("Sorry, the author of this video has marked it private, preventing it from being embedded.", function(){})
+          return
+        }
+        done({
+          url: url,
+          type: "vimeo",
+          token: id,
+          thumbnail: res.thumbnail_large,
+          duration: res.duration,
+          title: res.title,
+          width: res.width,
+          height: res.height,
+        })
+      })
+    },
+    tag: function (media) {
+      // return '<img class="video" type="vimeo" vid="'+media.token+'" src="'+media.thumbnail+'"><span class="playvid">&#9654;</span>';
+      return '<div class="video" style="width: ' + media.width + 'px; height: ' + media.height + 'px; overflow: hidden; position: relative;"><iframe frameborder="0" scrolling="no" seamless="seamless" webkitallowfullscreen="webkitAllowFullScreen" mozallowfullscreen="mozallowfullscreen" allowfullscreen="allowfullscreen" id="okplayer" src="https://player.vimeo.com/video/' + media.token + '?api=1&title=0&byline=0&portrait=0&playbar=0&player_id=okplayer&loop=0&autoplay=0" width="' + media.width + '" height="' + media.height + '" style="position: absolute; top: 0px; left: 0px; width: ' + media.width + 'px; height: ' + media.height + 'px;"></iframe></div>'
+    }
+  }, {
+    type: 'soundcloud',
+    regex: /soundcloud.com\/[-a-zA-Z0-9]+\/[-a-zA-Z0-9]+\/?$/i,
+    fetch: function (url, done) {
+      fetch('https://api.soundcloud.com/resolve.json?url=' 
+          + url
+          + '&client_id=' 
+          + '0673fbe6fc794a7750f680747e863b10')
+      .then(result => result.json())
+      .then(result => {
+          console.log(result)
+          done({
+            url: url,
+            type: "soundcloud",
+            token: result.id,
+            thumbnail: result.artwork_url || result.user.avatar_url,
+            title: result.user.username + " - " + result.title,
+            duration: result.duration,
+            width: 166,
+            height: 166,
+          })
+      })
+    },
+    tag: function (media) {
+      return '<iframe width="166" height="166" scrolling="no" frameborder="no"' +
+          'src="https://w.soundcloud.com/player/?url=https%3A//api.soundcloud.com/tracks/' + media.token +
+          '&amp;color=ff6600&amp;auto_play=false&amp;show_artwork=true"></iframe>'
+    }
+  },
+  {
+    type: 'link',
+    regex: /^http.+/i,
+    fetch: function(url, done) {
+      done({
+        url: url,
+        type: "link",
+        token: "",
+        thumbnail: "",
+        title: "",
+        width: 100,
+        height: 100,
+      })
+    },
+    tag: function (media) {
+      return '<a href="' + media.url + '" target="_blank">' + media.url + '</a>'
+    }
+  },
+]
+
+export const lookup = integrations.reduce((a,b) => (a[b.type] = b) && a, {})
+
+export const parse = (url, cb) => {
+  const matched = integrations.some(integration => {
+    if (integration.regex.test(url)) {
+      integration.fetch(url, function(res){
+        cb(res)
+      })
+      return true
+    }
+    return false
+  })
+  if (! matched) {
+    cb(null)
+  }
+}
+
+export const tag = media => {
+  if (media.type in lookup) {
+    return lookup[media.type].tag(media)
+  }
+  return ""
+}
+
+export const loadImage = (url, cb, error) => {
+  if (lookup.image.regex.test(url)) {
+    lookup.image.fetch(url, media => {
+      cb(media)
+    })
+  }
+  else error && error()
+}
+  
+export const thumbnail = media => {
+  return '<img src="' + (media.thumbnail || media.url) + '" class="thumb">';
+}
+
+export const tumblr = (url, cb) => {
+  let domain = url.replace(/^https?:\/\//,"").split("/")[0]
+  if (domain.indexOf(".") == -1) {
+    domain += ".tumblr.com"
+  }
+  fetchJsonp("http://" + domain + "/api/read")
+  .then(data => {
+    // const blog = data.tumblelog
+    const media_list = data.posts.reduce(parseTumblrPost, [])
+    cb(media_list)
+  })
+}
+
+function parseTumblrPost(media_list, post){
+  let media, caption, url
+  switch (post.type) {
+    case 'photo':
+      caption = stripHTML(post['photo-caption'])
+      if (post.photos.length) {
+        post.photos.forEach(function(photo){
+          media = {
+            url: photo['photo-url-1280'],
+            type: "image",
+            token: "",
+            thumbnail: photo['photo-url-500'],
+            description: caption,
+            width: parseInt(photo.width),
+            height: parseInt(photo.height),
+          }
+          media_list.push(media)
+        })
+      }
+      else {
+        media = {
+          url: post['photo-url-1280'],
+          type: "image",
+          token: "",
+          thumbnail: post['photo-url-500'],
+          description: caption,
+          width: parseInt(post.width),
+          height: parseInt(post.height),
+        }
+        media_list.push(media)
+      }
+      break
+    case 'video':
+      url = post['video-source']
+      if (url.indexOf("http") !== 0) { break }
+      if (lookup.youtube.regex.test(url)) {
+        const id = (url.match(/v=([-_a-zA-Z0-9]{11})/i) || url.match(/youtu.be\/([-_a-zA-Z0-9]{11})/i) || url.match(/embed\/([-_a-zA-Z0-9]{11})/i))[1].split('&')[0];
+        const thumb = "https://i.ytimg.com/vi/" + id + "/hqdefault.jpg"
+        media = {
+          url: post['video-source'],
+          type: "youtube",
+          token: id,
+          thumbnail: thumb,
+          title: stripHTML(post['video-caption']),
+          width: 640,
+          height: 360,
+        }
+        media_list.push(media)
+      }
+      break
+    default:
+      break
+  }
+  return media_list
+}
+
+
diff --git a/app/client/modules/samplernn/datasets.component.js b/app/client/modules/samplernn/datasets.component.js
index ae51c0f..441c17b 100644
--- a/app/client/modules/samplernn/datasets.component.js
+++ b/app/client/modules/samplernn/datasets.component.js
@@ -2,7 +2,7 @@ import { h, Component } from 'preact'
 import { bindActionCreators } from 'redux'
 import { connect } from 'react-redux'
 
-import { actions } from '../../api'
+import { actions, parser } from '../../api'
 import * as taskActions from '../../task/task.actions'
 import * as systemActions from '../../system/system.actions'
   // folderActions.index({ module: 'samplernn' })
@@ -72,15 +72,20 @@ class SampleRNNDatasets extends Component {
     // duration analysis
     // size activity
     // opt created_at updated_at
-    const folder = this.props.samplernn.folder
-    this.props.actions.file.create({
-      folder_id: folder.id,
-      module: 'samplernn',
-      activity: 'url',
-      epoch: 0,
-      processed: false,
-      generated: false,
-      url
+    parser.parse(url, media => {
+      if (!media) return
+      console.log('media', media)
+      return
+      const folder = this.props.samplernn.folder
+      this.props.actions.file.create({
+        folder_id: folder.id,
+        module: 'samplernn',
+        activity: 'url',
+        epoch: 0,
+        processed: false,
+        generated: false,
+        url
+      })
     })
   }
   fetchURL(url) {
@@ -122,6 +127,8 @@ class SampleRNNDatasets extends Component {
   }
   render(){
     const { samplernn } = this.props
+    console.log(samplernn.upload)
+    // sort files??
     return (
       <div className='app'>
         <div className='heading'>
@@ -153,13 +160,17 @@ class SampleRNNDatasets extends Component {
           </div>
         </div>
         <div className='params col'>
-          {samplernn.files.length ? 
-            <h3>Files</h3> :
-            <h4>No files</h4>}
+          <div class='row heading'>
+            {samplernn.files.length ? 
+              <h3>Files</h3> :
+              <h4>No files</h4>}
+            <div>{samplernn.upload.loading && samplernn.upload.status}</div>
+          </div>
           <FileList
             files={samplernn.files}
             options={this.fileOptions}
             onClick={this.pickFile}
+            showDelete
             linkFiles
           />
         </div>
diff --git a/app/client/modules/samplernn/samplernn.reducer.js b/app/client/modules/samplernn/samplernn.reducer.js
index ba429ee..efd31f5 100644
--- a/app/client/modules/samplernn/samplernn.reducer.js
+++ b/app/client/modules/samplernn/samplernn.reducer.js
@@ -8,8 +8,8 @@ const samplernnInitialState = {
   files: [],
   results: [],
   upload: {
-    loading: true,
-    status: 'Loading...',
+    loading: false,
+    status: '',
   },
 }
 
@@ -28,29 +28,21 @@ const samplernnReducer = (state = samplernnInitialState, action) => {
       return {
         ...state,
       }
+
     case types.folder.index:
-      console.log(action)
       return {
         ...state,
         folders: action.data,
         folder: action.data[0],
       }
     case types.folder.update:
-      console.log(action)
       return state
     case types.file.index:
-      console.log(action)
       return {
         ...state,
         files: action.data
       }
-    case types.file.create:
-      console.log(action)
-      return {
-        ...state,
-        files: [action.data].concat(this.files)
-      }
-      return
+
     case types.folder.upload_loading:
       return {
         ...state,
@@ -85,14 +77,43 @@ const samplernnReducer = (state = samplernnInitialState, action) => {
           status: 'Waiting for server to finish processing...',
         },
       }
+    case types.file.create_loading:
+      return {
+        ...state,
+        upload: {
+          loading: true,
+          status: 'Creating file...'
+        }
+      }
+    case types.file.create:
+    console.log('booo')
+      if (state.folder.id === action.data.folder_id) {
+        return {
+          ...state,
+          files: [action.data].concat(state.files),
+          upload: {
+            loading: false,
+            status: 'File created',
+          },
+        }
+      } else {
+        return {
+          ...state,
+          upload: {
+            loading: false,
+            status: 'created',
+          },
+        }
+      }
     case types.folder.upload_complete:
       console.log(action)
       if (state.folder.id === action.folder) {
         return {
           ...state,
-          files: state.files.concat(state.file), // sort here also
+          files: [action.files].concat(state.files),
           upload: {
             loading: false,
+            status: 'Upload complete',
           },
         }
       } else {