diff options
Diffstat (limited to 'scraper/client')
23 files changed, 1319 insertions, 0 deletions
diff --git a/scraper/client/actions.js b/scraper/client/actions.js new file mode 100644 index 00000000..ba899f06 --- /dev/null +++ b/scraper/client/actions.js @@ -0,0 +1,9 @@ +import * as search from './search/search.actions' +import * as review from './review/review.actions' +import * as metadata from './metadata/metadata.actions' + +export { + search, + review, + metadata, +} diff --git a/scraper/client/app.js b/scraper/client/app.js new file mode 100644 index 00000000..6c008ec6 --- /dev/null +++ b/scraper/client/app.js @@ -0,0 +1,46 @@ +import React, { Component } from 'react' +import { ConnectedRouter } from 'connected-react-router' +import { Route, Switch } from 'react-router' + +import { Header, Sidebar, Footer } from './common' +import * as Metadata from './metadata' +import * as Search from './search' +import * as Review from './review' + +export default class App extends Component { + render() { + return ( + <ConnectedRouter history={this.props.history}> + <div> + <Header /> + <div className='app'> + <Route path="/metadata/" component={Sidebar} /> + <div className='body'> + <Route path="/search/" component={Search.Menu} /> + <Route path="/metadata/:hash/" component={Metadata.Heading} /> + <Switch> + <Route exact path="/metadata/:hash/summary/" component={Metadata.Summary} /> + <Route exact path="/metadata/:hash/mediaRecord/" component={Metadata.MediaRecord} /> + <Route exact path="/metadata/:hash/mediaInfo/" component={Metadata.MediaInfo} /> + <Route exact path="/metadata/:hash/keyframe/:frame/" component={Metadata.KeyframeSingle} /> + <Route exact path="/metadata/:hash/keyframe/" component={Metadata.KeyframeList} /> + <Route exact path="/metadata/:hash/coco/" component={Metadata.Coco} /> + <Route exact path="/metadata/:hash/places365/" component={Metadata.Places365} /> + <Route exact path="/metadata/:hash/sugarcube/" component={Metadata.Sugarcube} /> + <Route exact path="/metadata/:hash/" component={Metadata.Summary} /> + <Route exact path="/metadata/" render={() => <div className='notFound'><h4>NOT FOUND</h4></div>} /> + <Route exact path="/search/" component={Search.Container} /> + <Route exact path="/search/keyframe/:verified/:hash/:frame/" component={Search.Container} /> + <Route exact path="/search/keyframe/:hash/:frame/" component={Search.Container} /> + <Route exact path="/search/browse/:hash/" component={Search.Browse} /> + <Route exact path="/search/random/" component={Search.Random} /> + <Route exact path="/search/review/" component={Review.Saved} /> + </Switch> + </div> + </div> + <Footer /> + </div> + </ConnectedRouter> + ) + } +} diff --git a/scraper/client/common/activeLink.component.js b/scraper/client/common/activeLink.component.js new file mode 100644 index 00000000..59f63881 --- /dev/null +++ b/scraper/client/common/activeLink.component.js @@ -0,0 +1,16 @@ +import React from 'react' +import { NavLink } from 'react-router-dom' + +export default function ActiveLink({ + to, + className = 'navlink', + children +}) { + return ( + <span className={className}> + <NavLink to={to}> + {children} + </NavLink> + </span> + ) +} diff --git a/scraper/client/common/classifier.component.js b/scraper/client/common/classifier.component.js new file mode 100644 index 00000000..af6a4934 --- /dev/null +++ b/scraper/client/common/classifier.component.js @@ -0,0 +1,99 @@ +import React, { Component } from 'react' +import { courtesyS } from '../util' + +import { TableTuples, DetectionList, Keyframe } from '.' + +export default class Classifier extends Component { + render() { + const { + tag, + sha256, + verified, + keyframes = {}, + labels, + summary, + aspectRatio = 1.777, + showAll, + } = this.props + let totalDetections = 0 + const keys = Object + .keys(keyframes) + .map(s => parseInt(s, 10)) + const validKeyframes = keys + .sort((a, b) => a - b) + .map(frame => { + const detections = keyframes[frame] + if (detections.length || showAll) { + totalDetections += detections.length + return { frame, detections } + } + return null + }) + .filter(f => !!f) + const detectionLookup = validKeyframes + .reduce((a, b) => { + b.detections.reduce((aa, { idx }) => { + if (!(idx in aa)) aa[idx] = [labels[idx], 0] + aa[idx][1] += 1 + return aa + }, a) + return a + }, {}) + const detectionCounts = Object.keys(detectionLookup) + .map(idx => detectionLookup[idx]) + .sort((a, b) => b[1] - a[1]) + + if (summary) { + return ( + <div> + <h3>{tag}{' Detections'}</h3> + <TableTuples + list={detectionCounts} + /> + </div> + ) + } + return ( + <div> + <h2>{tag}</h2> + <h3>Detections</h3> + <TableTuples + list={detectionCounts} + /> + <h3>Frames</h3> + <ul className='meta'> + <li> + {'Displaying '}{validKeyframes.length}{' / '}{courtesyS(keys.length, 'frame')} + </li> + <li> + {courtesyS(totalDetections, 'detection')}{' found'} + </li> + </ul> + <div className='thumbnails'> + {validKeyframes.map(({ frame, detections }) => ( + <Keyframe + key={frame} + sha256={sha256} + frame={frame} + verified={verified} + size='th' + showFrame + showTimestamp + aspectRatio={aspectRatio} + detectionList={[ + { labels, detections } + ]} + > + <DetectionList + labels={labels} + detections={detections} + width={160} + height={90} + /> + </Keyframe> + ))} + </div> + </div> + ) + } +} diff --git a/scraper/client/common/common.css b/scraper/client/common/common.css new file mode 100644 index 00000000..4b939df0 --- /dev/null +++ b/scraper/client/common/common.css @@ -0,0 +1,347 @@ +/* css boilerplate */ + +* { box-sizing: border-box; } +html,body { + margin: 0; padding: 0; + width: 100%; height: 100%; +} +body { + font-family: Helvetica, sans-serif; + font-weight: 300; +} + +h1 { + +} +h2 { + font-weight: normal; + margin: 10px 0; + padding: 3px; + font-size: 24px; +} +h3 { + font-weight: normal; + margin: 10px 0 0 0; + padding: 3px; + font-size: 18px; +} +h4 { + font-weight: 300; + font-size: 12px; + letter-spacing: 2px; + color: #888; + text-transform: uppercase; + margin: 5px 10px; + margin-top: 20px; +} +h4:first-child { + margin-top: 10px; +} + +.app { + width: 100%; + height: 100%; + display: flex; + flex-direction: row; + align-items: flex-start; + justify-content: flex-start; +} + +/* header stuff */ + +header { + width: 100%; + background: #11f; + color: white; + align-items: stretch; + display: flex; + flex-wrap: wrap; + justify-content: space-between; + z-index: 3; +} +header > section { + justify-content: flex-start; + align-items: center; + display: flex; + flex: 1 0; + font-weight: bold; +} +header > section:last-of-type { + justify-content: flex-end; +} + +/* sidebar / body columns */ + +.sidebar { + display: flex; + flex-direction: column; + justify-content: flex-start; + align-items: flex-start; + height: 100%; + float: left; + width: 200px; + flex: 0 0 200px; + padding: 10px; + margin-right: 10px; +} +.sidebar a { + display: block; + padding: 10px 10px; + text-decoration: none; + color: #444; +} +.sidebar a.active { + font-weight: bold; + color: #222; +} +.body { + display: flex; + flex-direction: column; + align-items: flex-start; + justify-content: flex-start; + flex-grow: 1; +} +.body > div { + padding-bottom: 40px; +} + +/* buttons / forms */ + +.btn:focus, .btn:hover { + background: #f1f1fc; + color: #4b48d6 !important; + text-decoration: none; +} +.btn { + -webkit-appearance: none; + -moz-appearance: none; + appearance: none; + background: #fff; + border: .05rem solid; + border-radius: 2px; + margin-right: 5px; + color: #11f; + cursor: pointer; + display: inline-block; + font-size: .8rem; + height: 1.8rem; + line-height: 1rem; + outline: none; + padding: .35rem .4rem; + text-align: center; + text-decoration: none; + -webkit-transition: all .2s ease; + -o-transition: all .2s ease; + transition: all .2s ease; + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; + vertical-align: middle; + white-space: nowrap; +} +.btn.reset, +.btn.panic { + color: #b00; +} +.btn.btn-primary { + background: #11f; + border-color: #11f; + color: white; +} +.btn[disabled] { + color: #bbb !important; + border-color: #bbb !important; + background: white !important; + cursor: default; +} +.btn.btn-primary:focus, +.btn.btn-primary:hover { + background: #0808ee; + color: white !important; +} +.row .btn { + margin: 0 5px 0 0; +} +input[type=text] { + border: 1px solid #888; + padding: 4px; + font-size: 15px; +} + + +/* tables on metadata pages */ + +table { + border: 0; + margin: 0; + padding: 0; + border-spacing: 0; +} +.tableObject td, +.tableObject th { + padding: 3px; + vertical-align: top; +} +.tableObject hr { + width: 100%; + color: transparent; + border: 0; + border-bottom: 1px solid #bbb; + align: left; + margin: 3px 0; + padding: 0; +} +.tableObject th, +.tableTuples th { + min-width: 145px; + text-align: left; + text-transform: capitalize; + padding: 3px; + padding-right: 10px; + font-weight: 300; + color: #333; +} +.tableTuples td { + text-align: right; + padding: 3px; +} +.tableObject td { + font-weight: normal; + color: #000; +} +.tableObject .tableObject { + border: 1px solid #ddd; +} +.tableArray { + border: 1px solid #ddd; + border-spacing: 0; +} +.tableArray td { + border-bottom: 1px solid #ddd; +} +.gray { + font-size: 12px; + color: #888; + display: block; +} +.sha256.heading { + margin: 20px 0 0px; +} +.gray span { + padding-right: 5px; +} +.gray { + margin-bottom: 10px; +} +.gray a { + color: #666; +} + +.verified { + color: #080; + font-weight: bold; +} +.unverified { + color: #f00; + font-weight: 300; +} + +.loading, .error { + font-weight: normal; + margin: 10px 0; + padding: 3px; + font-size: 24px; +} + +.title { + text-transform: capitalize; +} +.rect { + position: absolute; +} +.rect { border: 1px solid rgba(0,0,255); background-color: rgba(0,0,255,0.1); } + +/* videos / video preloader */ + +video { + max-width: 640px; + margin: 10px 0; +} +.video { + margin: 0 0 10px 0; +} +.video .bg { + cursor: pointer; + position: relative; + background-size: cover; +} +.video .play { + position: absolute; + top: 50%; + left: 50%; + transform: translate3d(-50%, -50%, 0); + width: 20%; + height: 20%; + background-image: url(/search/static/img/play.png); + background-position: center center; + background-size: contain; + background-repeat: no-repeat; +} +.desktop .video .play:hover { + -webkit-filter: invert(60%) sepia(100%) saturate(500%) hue-rotate(160deg); +} + +/* spectre.css loader */ + +.loaderWrapper { + display: inline-block; + position: relative; + width: .8rem; + height: .8rem; + padding: 10px; +} +.loader { + color: transparent !important; + min-height: .8rem; + pointer-events: none; + position: relative; +} + +.loader::after { + animation: loader 500ms infinite linear; + border: .1rem solid #5755d9; + border-radius: 50%; + border-right-color: transparent; + border-top-color: transparent; + content: ""; + display: block; + height: .8rem; + left: 50%; + margin-left: -.4rem; + margin-top: -.4rem; + position: absolute; + top: 50%; + width: .8rem; + z-index: 1; +} + +.loader.loader-lg { + min-height: 2rem; +} + +.loader.loader-lg::after { + height: 1.6rem; + margin-left: -.8rem; + margin-top: -.8rem; + width: 1.6rem; +} + +@keyframes loader { + 0% { + transform: rotate(0deg); + } + 100% { + transform: rotate(360deg); + } +}
\ No newline at end of file diff --git a/scraper/client/common/detectionBoxes.component.js b/scraper/client/common/detectionBoxes.component.js new file mode 100644 index 00000000..c4872ea8 --- /dev/null +++ b/scraper/client/common/detectionBoxes.component.js @@ -0,0 +1,15 @@ +import React from 'react' + +import { px } from '../util' + +export default function DetectionBoxes({ detections, width, height }) { + return detections.map(({ rect }, i) => ( + rect && + <div className='rect' key={i} style={{ + left: px(rect[0], width), + top: px(rect[1], height), + width: px(rect[2] - rect[0], width), + height: px(rect[3] - rect[1], height), + }} /> + )) +} diff --git a/scraper/client/common/detectionList.component.js b/scraper/client/common/detectionList.component.js new file mode 100644 index 00000000..416e66d8 --- /dev/null +++ b/scraper/client/common/detectionList.component.js @@ -0,0 +1,16 @@ +import React from 'react' + +export default function DetectionList({ detections, labels, tag, showEmpty }) { + return ( + <span className='detectionList'> + {tag && <h3>{tag}</h3>} + {!detections.length && showEmpty && <label><small>No detections</small></label>} + {detections.map(({ idx, score, rect }, i) => ( + <label key={i}> + <small className='title'>{(labels[idx] || 'Unknown').replace(/_/, ' ')}</small> + <small>{score.toFixed(2)}</small> + </label> + ))} + </span> + ) +} diff --git a/scraper/client/common/footer.component.js b/scraper/client/common/footer.component.js new file mode 100644 index 00000000..7c82b44b --- /dev/null +++ b/scraper/client/common/footer.component.js @@ -0,0 +1,10 @@ +import React from 'react' +import { Link } from 'react-router-dom' +import { connect } from 'react-redux' + +export default function Footer(props) { + return ( + <footer> + </footer> + ); +} diff --git a/scraper/client/common/gate.component.js b/scraper/client/common/gate.component.js new file mode 100644 index 00000000..9bf9287b --- /dev/null +++ b/scraper/client/common/gate.component.js @@ -0,0 +1,21 @@ +import React from 'react' +import { connect } from 'react-redux' + +function Gate(props) { + const { app, tag, View } = props + const data = app[tag] + if (!data) return null + if (data === 'loading') { + return <div className='tableObject loading'>{tag}{': Loading'}</div> + } + if (data.err) { + return <div className='tableObject error'>{tag}{' Error: '}{data.err}</div> + } + return <View data={data} {...props} /> +} + +const mapStateToProps = state => ({ + app: state.metadata +}) + +export default connect(mapStateToProps)(Gate) diff --git a/scraper/client/common/header.component.js b/scraper/client/common/header.component.js new file mode 100644 index 00000000..84fe306f --- /dev/null +++ b/scraper/client/common/header.component.js @@ -0,0 +1 @@ +/* imported from main vcat application */ diff --git a/scraper/client/common/index.js b/scraper/client/common/index.js new file mode 100644 index 00000000..ad9fe5e1 --- /dev/null +++ b/scraper/client/common/index.js @@ -0,0 +1,36 @@ +import Header from 'vcat-header' + +import ActiveLink from './activeLink.component' +import Classifier from './classifier.component' +import DetectionBoxes from './detectionBoxes.component' +import DetectionList from './detectionList.component' +// import Header from './header.component' +import Footer from './footer.component' +import Loader from './loader.component' +import Sidebar from './sidebar.component' +import Gate from './gate.component' +import Keyframe from './keyframe.component' +import Keyframes from './keyframes.component' +import Video from './video.component' +import { TableObject, TableArray, TableTuples, TableRow, TableCell } from './table.component' +import './common.css' + +export { + Header, + Footer, + Sidebar, + Loader, + Gate, + TableObject, + TableArray, + TableTuples, + TableRow, + TableCell, + ActiveLink, + Classifier, + DetectionList, + DetectionBoxes, + Keyframe, + Keyframes, + Video, +} diff --git a/scraper/client/common/keyframe.component.js b/scraper/client/common/keyframe.component.js new file mode 100644 index 00000000..c77db3ac --- /dev/null +++ b/scraper/client/common/keyframe.component.js @@ -0,0 +1,118 @@ +import React from 'react' +import { Link } from 'react-router-dom' +import { imageUrl, timestamp, keyframeUri, widths, verify } from '../util' +import { DetectionBoxes } from '.' + +import * as searchActions from '../search/search.actions' + +export default function Keyframe({ + verified, + sha256, + frame, + score, + isSaved, + fps = 25, + size = 'th', + className, + showHash, + showFrame, + showTimestamp, + showScore, + showSearchButton, + showSaveButton, + to, + children, + detectionList = [], + aspectRatio = 1.777, + onClick, + reviewActions, +}) { + if (!sha256) return null + const width = widths[size] + const height = Math.round(width / aspectRatio) + return ( + <div className={(className || 'keyframe') + (isSaved ? ' isSaved' : '')}> + <div className="thumbnail"> + <PossiblyExternalLink to={to || keyframeUri(sha256, frame)} onClick={onClick}> + <img + alt={'Frame #' + frame} + src={imageUrl(verified, sha256, frame, size)} + width={width} + height={height} + onClick={onClick} + /> + {detectionList.map(({ labels, detections }, i) => ( + <DetectionBoxes + key={i} + labels={labels} + detections={detections} + width={width} + height={height} + /> + ))} + </PossiblyExternalLink> + {(reviewActions && (showSearchButton || showSaveButton)) && + <label className='searchButtons'> + {showSearchButton && + <Link + to={searchActions.publicUrl.searchByVerifiedFrame(verified, sha256, frame)} + className='btn' + > + Search + </Link> + } + {showSaveButton && (isSaved + ? <button + onClick={() => reviewActions.unsave({ hash: sha256, frame, verified })} + className={'btn btn-primary saved'} + > + {'Saved'} + </button> + : <button + onClick={() => reviewActions.save({ hash: sha256, frame, verified })} + className={'btn btn save'} + > + {'Save'} + </button> + )} + </label> + } + </div> + {(showHash || showFrame || showTimestamp || showScore) && + <label> + {showHash && + <small> + <Link to={searchActions.publicUrl.browse(sha256)}> + <span + title={sha256} + className={'sha256 ' + verify(verified)} + > + {'▶ '} + {sha256.substr(0, 6)} + </span> + </Link> + </small> + } + {showFrame && + <small> + <span>{'Frame #'}{frame}</span> + </small> + } + {showTimestamp && <small>{timestamp(frame, fps)}</small>} + {showScore && !!score && <small>{score}</small>} + </label> + } + {children} + </div> + ) +} + +const PossiblyExternalLink = props => { + if (props.onClick) { + return props.children + } + if (props.to.match(/^http/)) { + return <a href={props.to} target='_blank' rel='noopener noreferrer'>{props.children}</a> + } + return <Link {...props} /> +} diff --git a/scraper/client/common/keyframes.component.js b/scraper/client/common/keyframes.component.js new file mode 100644 index 00000000..62eda45e --- /dev/null +++ b/scraper/client/common/keyframes.component.js @@ -0,0 +1,95 @@ +import React from 'react' +import { Link } from 'react-router-dom' +import { bindActionCreators } from 'redux' +import { connect } from 'react-redux' + +import { Keyframe } from '.' +import * as reviewActions from '../review/review.actions' +import * as searchActions from '../search/search.actions' + +function Keyframes(props) { + // console.log(props) + let { + frames, + groupByHash, + } = props + let minDistance = 0 + if (frames && frames.length) { + minDistance = frames[0].distance || 0 + } + if (!groupByHash) { + return ( + <KeyframeList + minDistance={minDistance} + {...props} + /> + ) + } + const frameGroups = frames.reduce((a, b) => { + if (a[b.hash]) { + a[b.hash].push(b) + } else { + a[b.hash] = [b] + } + return a + }, {}) + return Object.keys(frameGroups) + .map(hash => [frameGroups[hash].length, hash]) + .sort((a, b) => b[0] - a[0]) + .map(([count, hash]) => ( + <KeyframeList + {...props} + count={count} + key={hash} + minDistance={minDistance} + frames={frameGroups[hash]} + label={hash} + /> + )) +} + +function KeyframeList(props) { + let { + saved = {}, + frames, + options, + review, + search, + minDistance, + label, + count, + ...frameProps + } = props + if (!frames) return null + return ( + <div className={label ? 'keyframes keyframeGroup' : 'keyframes'}> + {label && <h4><Link to={searchActions.publicUrl.browse(label)}>{label}</Link> ({count})</h4>} + {frames.map(({ hash, frame, verified, distance }) => ( + <Keyframe + key={hash + '_' + frame} + sha256={hash} + frame={frame} + score={100 - Math.round(distance - minDistance) + '%'} + verified={verified} + isSaved={!!saved[hash] && !!saved[hash].frames && !!saved[hash].frames[parseInt(frame, 10)]} + size={options.thumbnailSize} + onClick={() => review.toggleSaved({ verified, hash, frame })} + reviewActions={review} + {...frameProps} + /> + ))} + </div> + ) +} + +const mapStateToProps = state => ({ + saved: state.review.saved, + options: state.search.options, +}) + +const mapDispatchToProps = dispatch => ({ + review: bindActionCreators({ ...reviewActions }, dispatch), + search: bindActionCreators({ ...searchActions }, dispatch), +}) + +export default connect(mapStateToProps, mapDispatchToProps)(Keyframes) diff --git a/scraper/client/common/loader.component.js b/scraper/client/common/loader.component.js new file mode 100644 index 00000000..6795424b --- /dev/null +++ b/scraper/client/common/loader.component.js @@ -0,0 +1,10 @@ +import React, { Component } from 'react' + +export default function Loader() { + return ( + <div className='loaderWrapper'> + <div className='loader'> + </div> + </div> + ) +}
\ No newline at end of file diff --git a/scraper/client/common/sidebar.component.js b/scraper/client/common/sidebar.component.js new file mode 100644 index 00000000..487f3289 --- /dev/null +++ b/scraper/client/common/sidebar.component.js @@ -0,0 +1,37 @@ +import React, { Component } from 'react' +import { NavLink } from 'react-router-dom' +import { connect } from 'react-redux' + +class Sidebar extends Component { + render() { + const { hash } = this.props + if (!hash) { + return ( + <div className="sidebar"> + </div> + ) + } + return ( + <div className="sidebar"> + <h4>Media</h4> + <NavLink to={'/metadata/' + hash + '/summary/'}>Summary</NavLink> + <NavLink to={'/metadata/' + hash + '/mediaRecord/'}>Media Record</NavLink> + <NavLink to={'/metadata/' + hash + '/mediaInfo/'}>Media Info</NavLink> + <NavLink to={'/metadata/' + hash + '/sugarcube/'}>Sugarcube</NavLink> + + <h4>Keyframes</h4> + <NavLink to={'/metadata/' + hash + '/keyframe/'}>Keyframe</NavLink> + + <h4>Detectors</h4> + <NavLink to={'/metadata/' + hash + '/places365/'}>Places 365</NavLink> + <NavLink to={'/metadata/' + hash + '/coco/'}>Coco</NavLink> + </div> + ) + } +} + +const mapStateToProps = state => ({ + hash: state.metadata.hash, +}) + +export default connect(mapStateToProps)(Sidebar) diff --git a/scraper/client/common/table.component.js b/scraper/client/common/table.component.js new file mode 100644 index 00000000..76a1d57c --- /dev/null +++ b/scraper/client/common/table.component.js @@ -0,0 +1,121 @@ +import React from 'react' + +import { formatName } from '../util' + +const __HR__ = '__HR__' + +export function TableObject({ tag, object, order, summary }) { + if (!object) return null + if (object === 'loading') { + return <div className='tableObject loading'>{tag}{': Loading'}</div> + } + if (object.err) { + return <div className='tableObject error'>{tag}{' Error: '}{object.err}</div> + } + let objects = Object.keys(object) + if (order) { + const grouped = objects.reduce((a, b) => { + const index = order.indexOf(b) + if (index !== -1) { + a.order.push([index, b]) + } else { + a.alpha.push(b) + } + return a + }, { order: [], alpha: [] }) + objects = grouped.order + .sort((a, b) => a[0] - b[0]) + .map(([i, s]) => s) + if (!summary) { + objects = objects + // .concat([__HR__]) + .concat(grouped.alpha.sort()) + } + } else { + objects = objects.sort() + } + return ( + <div> + {tag && <h3>{tag}</h3>} + <table className={'tableObject ' + tag}> + <tbody> + {objects.map((key, i) => ( + <TableRow key={key + '_' + i} name={key} value={object[key]} /> + ))} + </tbody> + </table> + </div> + ) +} + +export function TableArray({ tag, list }) { + if (!list) return null + return ( + <div> + {tag && <h3>{tag}</h3>} + <table className={'tableArray ' + tag}> + <tbody> + {list.map((value, i) => ( + <tr key={tag + '_' + i}> + <TableCell value={value} /> + </tr> + ))} + </tbody> + </table> + </div> + ) +} + +export function TableTuples({ tag, list }) { + if (!list) return null + return ( + <div> + {tag && <h3>{tag}</h3>} + <table className={'tableTuples ' + tag}> + <tbody> + {list.map(([key, ...values], i) => ( + <tr key={tag + '_' + i}> + <th>{formatName(key)}</th> + {values.map((value, j) => ( + <TableCell key={i + '_' + j} value={value} /> + ))} + </tr> + ))} + </tbody> + </table> + </div> + ) +} + +export function TableRow({ name, value }) { + if (name === __HR__) { + return ( + <tr> + <th className='tr'> + <hr /> + </th> + </tr> + ) + } + return ( + <tr> + <th>{formatName(name)}</th> + <TableCell name={name} value={value} /> + </tr> + ) +} + +export function TableCell({ value }) { + if (value && typeof value === 'object') { + if (value._raw) { + value = value.value + } else if (value.length) { + value = <TableArray nested tag={''} list={value} /> + } else { + value = <TableObject nested tag={''} object={value} /> + } + } + return ( + <td>{value}</td> + ) +} diff --git a/scraper/client/common/video.component.js b/scraper/client/common/video.component.js new file mode 100644 index 00000000..e5525bf6 --- /dev/null +++ b/scraper/client/common/video.component.js @@ -0,0 +1,47 @@ +import React, { Component } from 'react' +import { connect } from 'react-redux' +import { imageUrl, widths } from '../util' + +import { Gate } from '.' + +class Video extends Component { + state = { + playing: false, + } + + render() { + const { app, data, size } = this.props + const { playing } = this.state + const { sugarcube } = data.metadata + const url = sugarcube.fp.replace('/var/www/files/', 'https://cube.syrianarchive.org/') + const { sha256, verified } = app.mediainfo + const { video } = app.mediainfo.metadata.mediainfo + const keyframe = app.keyframe.metadata.keyframe.basic[0] + return ( + <div className='video'> + {playing + ? <video src={url} autoPlay controls muted /> + : <div + className='bg' + style={{ + width: widths[size || 'sm'], + height: widths[size || 'sm'] / video.aspect_ratio, + backgroundImage: 'url(' + imageUrl(verified, sha256, keyframe, size) + ')', + }} + onClick={() => this.setState({ playing: true })} + > + <div className='play'></div> + </div> + } + </div> + ) + } +} + +const mapStateToProps = () => ({ + tag: 'sugarcube', +}) + +export default connect(mapStateToProps)(props => ( + <Gate View={Video} {...props} /> +)) diff --git a/scraper/client/index.js b/scraper/client/index.js new file mode 100644 index 00000000..eddc5fb2 --- /dev/null +++ b/scraper/client/index.js @@ -0,0 +1,19 @@ +import React from 'react' +import ReactDOM from 'react-dom' +import { AppContainer } from 'react-hot-loader' +import { Provider } from 'react-redux' + +import App from './app' + +import { store, history } from './store' + +const container = document.createElement('div') +document.body.appendChild(container) + +ReactDOM.render( + <AppContainer> + <Provider store={store}> + <App history={history} /> + </Provider> + </AppContainer>, container +) diff --git a/scraper/client/metadata/index.js b/scraper/client/metadata/index.js new file mode 100644 index 00000000..0eef814e --- /dev/null +++ b/scraper/client/metadata/index.js @@ -0,0 +1,25 @@ +import Heading from './heading.component' +import MediaInfo from './mediaInfo.component' +import MediaRecord from './mediaRecord.component' +import Summary from './summary.component' +import KeyframeList from './keyframeList.component' +import KeyframeSingle from './keyframeSingle.component' +import KeyframeStatus from './keyframeStatus.component' +import Coco from './coco.component' +import Places365 from './places365.component' +import Sugarcube from './sugarcube.component' + +import './metadata.css' + +export { + Heading, + MediaRecord, + MediaInfo, + Summary, + KeyframeList, + KeyframeSingle, + KeyframeStatus, + Coco, + Places365, + Sugarcube, +} diff --git a/scraper/client/session.js b/scraper/client/session.js new file mode 100644 index 00000000..5bfae7eb --- /dev/null +++ b/scraper/client/session.js @@ -0,0 +1,5 @@ +import Storage from 'store2' + +const session = Storage.namespace('vcat.search') + +export default session diff --git a/scraper/client/store.js b/scraper/client/store.js new file mode 100644 index 00000000..043af351 --- /dev/null +++ b/scraper/client/store.js @@ -0,0 +1,38 @@ +import { applyMiddleware, compose, combineReducers, createStore } from 'redux' +import { connectRouter, routerMiddleware } from 'connected-react-router' +import { createBrowserHistory } from 'history' +import thunk from 'redux-thunk' +import { login } from './util' + +import metadataReducer from './metadata/metadata.reducer' +import searchReducer from './search/search.reducer' +import reviewReducer from './review/review.reducer' + +const rootReducer = combineReducers({ + auth: (state = login()) => state, + metadata: metadataReducer, + search: searchReducer, + review: reviewReducer, +}) + +function configureStore(initialState = {}, history) { + const composeEnhancers = window.__REDUX_DEVTOOLS_EXTENSION_COMPOSE__ || compose + + const store = createStore( + connectRouter(history)(rootReducer), // new root reducer with router state + initialState, + composeEnhancers( + applyMiddleware( + thunk, + routerMiddleware(history) + ), + ), + ) + + return store +} + +const history = createBrowserHistory() +const store = configureStore({}, history) + +export { store, history } diff --git a/scraper/client/types.js b/scraper/client/types.js new file mode 100644 index 00000000..e3c64691 --- /dev/null +++ b/scraper/client/types.js @@ -0,0 +1,21 @@ +export const asType = (type, name) => [type, name].join('_').toUpperCase() +export const tagAsType = (type, names) => ( + names.reduce((tags, name) => { + tags[name] = asType(type, name) + return tags + }, {}) +) + +export const metadata = tagAsType('metadata', [ + 'loading', 'loaded', 'loaded_many', 'error', 'set_hash' +]) + +export const search = tagAsType('search', [ + 'loading', 'loaded', 'error', 'panic', 'update_options', +]) + +export const review = tagAsType('review', [ + 'loading', 'loaded', 'error', 'save', 'unsave', 'refresh', 'clear', 'dedupe', 'create', 'set_count' +]) + +export const init = '@@INIT' diff --git a/scraper/client/util.js b/scraper/client/util.js new file mode 100644 index 00000000..ad303c64 --- /dev/null +++ b/scraper/client/util.js @@ -0,0 +1,167 @@ +/* Mobile check */ + +export const isiPhone = !!((navigator.userAgent.match(/iPhone/i)) || (navigator.userAgent.match(/iPod/i))) +export const isiPad = !!(navigator.userAgent.match(/iPad/i)) +export const isAndroid = !!(navigator.userAgent.match(/Android/i)) +export const isMobile = isiPhone || isiPad || isAndroid +export const isDesktop = !isMobile + +const htmlClassList = document.body.parentNode.classList +htmlClassList.add(isDesktop ? 'desktop' : 'mobile') + +/* Default image dimensions */ + +export const widths = { + th: 160, + sm: 320, + md: 640, + lg: 1280, +} + +/* Formatting functions */ + +const acronyms = 'id url cc sa fp md5 sha256'.split(' ').map(s => '_' + s) +const acronymsUpperCase = acronyms.map(s => s.toUpperCase()) + +export const formatName = s => { + acronyms.forEach((acronym, i) => s = s.replace(acronym, acronymsUpperCase[i])) + return s.replace(/_/g, ' ') +} + +// Use to pad frame numbers with zeroes +export const pad = (n, m) => { + let s = String(n || 0) + while (s.length < m) { + s = '0' + s + } + return s +} + +// Verified is 0/1 when retrieved from SQL, but 'verified' or 'unverified' when retrieved elsewhere +export const isVerified = verified => verified === 1 || verified === '1' || verified === 'verified' +export const verify = verified => isVerified(verified) ? 'verified' : 'unverified' + +export const courtesyS = (n, s) => n + ' ' + (n === 1 ? s : s + 's') + +export const padSeconds = n => n < 10 ? '0' + n : n + +export const timestamp = (n = 0, fps = 25) => { + n /= fps + let s = padSeconds(Math.round(n) % 60) + n = Math.floor(n / 60) + if (n > 60) { + return Math.floor(n / 60) + ':' + padSeconds(n % 60) + ':' + s + } + return (n % 60) + ':' + s +} + +export const percent = n => (n * 100).toFixed(1) + '%' + +export const px = (n, w) => Math.round(n * w) + 'px' + +export const clamp = (n, a, b) => n < a ? a : n < b ? n : b + +/* URLs */ + +export const hashPath = sha256 => { + if (!sha256 || sha256.length < 9) { + throw new Error('Invalid sha256') + } + return [ + sha256.slice(0, 3), + sha256.slice(3, 6), + sha256.slice(6, 9), + sha256, + ].join('/') +} + +export const imageUrl = (verified, sha256, frame, size = 'th') => [ + 'https://' + process.env.S3_HOST + '/v1/media/keyframes', + isVerified(verified) ? null : 'unverified', + hashPath(sha256), + pad(frame, 6), + size, + 'index.jpg' +].filter(s => !!s).join('/') + +export const metadataUri = (sha256, tag) => '/metadata/' + sha256 + '/' + tag + '/' +export const keyframeUri = (sha256, frame) => '/metadata/' + sha256 + '/keyframe/' + pad(frame, 6) + '/' + +export const preloadImage = opt => { + let { verified, hash, frame, url } = opt + if (hash && frame) { + url = imageUrl(verified, hash, frame, 'md') + } + const image = new Image() + let loaded = false + image.onload = () => { + if (loaded) return + loaded = true + image.onload = null + } + // console.log(img.src) + image.crossOrigin = 'anonymous' + image.src = url + if (image.complete) { + image.onload() + } +} + +/* AJAX */ + +let cachedAuth = null +let token = '' +let username = '' + +export const post = (uri, data, credentials) => { + login() + let headers + if (data instanceof FormData) { + headers = { + Accept: 'application/json, application/xml, text/play, text/html, *.*', + } + } else { + headers = { + Accept: 'application/json, application/xml, text/play, text/html, *.*', + 'Content-Type': 'application/json; charset=utf-8', + } + data = JSON.stringify(data) + } + let opt = { + method: 'POST', + body: data, + headers, + credentials: 'include', + } + if (credentials) { + headers.Authorization = 'Token ' + token + } + // console.log(headers) + // headers['X-CSRFToken'] = csrftoken + return fetch(uri, opt).then(res => res.json()) +} + +// api queries +export const login = () => { + if (cachedAuth) return cachedAuth + const isLocal = (window.location.hostname === '0.0.0.0' || window.location.hostname === '127.0.0.1') + try { + const auth = JSON.parse(JSON.parse(localStorage.getItem('persist:root')).auth) + // console.log('auth', auth) + token = auth.token + username = auth.user.username + if (token) { + console.log('logged in', username) + } + cachedAuth = auth + if (!token && !isLocal) { + window.location.href = '/' + } + return auth + } catch (e) { + if (!isLocal) { + window.location.href = '/' + } + return {} + } +} |