diff options
29 files changed, 2090 insertions, 0 deletions
diff --git a/scraper/.gitignore b/scraper/.gitignore index 868c3dd4..f1ce9662 100644 --- a/scraper/.gitignore +++ b/scraper/.gitignore @@ -2,3 +2,5 @@ datasets/s2 datasets/old datasets/scholar_entries.numbers datasets/scholar_entries.csv +reports/geocode-app.js + diff --git a/scraper/client/actions.js b/scraper/client/actions.js new file mode 100644 index 00000000..f281a34c --- /dev/null +++ b/scraper/client/actions.js @@ -0,0 +1,23 @@ +import { get, post } from './util' +import * as types from './types' + +export const api = (dispatch, method, tag, url, params) => { + dispatch({ type: types.api.loading, tag }) + get(url, params).then(data => { + dispatch({ type: types.api.loaded, tag, data }) + }).catch(err => { + dispatch({ type: types.api.error, tag, err }) + }) +} + +export const getInstitutions = () => dispatch => { + api(dispatch, get, 'institutions', '/api/institutions', {}) +} + +export const getPapers = () => dispatch => { + api(dispatch, get, 'papers', '/api/papers', {}) +} + +export const postAddress = data => dispatch => { + api(dispatch, post, 'address', '/api/address', data) +} diff --git a/scraper/client/app.js b/scraper/client/app.js new file mode 100644 index 00000000..a115041b --- /dev/null +++ b/scraper/client/app.js @@ -0,0 +1,47 @@ +import React, { Component } from 'react' +import { ConnectedRouter } from 'connected-react-router' +import { Route, Switch } from 'react-router' + +import { Header, Sidebar, Footer } from './common' +// import * as Metadata from './metadata' +// import * as Search from './search' +// import * as Review from './review' + +export default class App extends Component { + render() { + return ( + <ConnectedRouter history={this.props.history}> + <div> + <Header /> + <div className='app'> + <Route path="/metadata/" component={Sidebar} /> + <div className='body'> + </div> + </div> + <Footer /> + </div> + </ConnectedRouter> + ) + } +} + + // <Route path="/search/" component={Search.Menu} /> + // <Route path="/metadata/:hash/" component={Metadata.Heading} /> + // <Switch> + // <Route exact path="/metadata/:hash/summary/" component={Metadata.Summary} /> + // <Route exact path="/metadata/:hash/mediaRecord/" component={Metadata.MediaRecord} /> + // <Route exact path="/metadata/:hash/mediaInfo/" component={Metadata.MediaInfo} /> + // <Route exact path="/metadata/:hash/keyframe/:frame/" component={Metadata.KeyframeSingle} /> + // <Route exact path="/metadata/:hash/keyframe/" component={Metadata.KeyframeList} /> + // <Route exact path="/metadata/:hash/coco/" component={Metadata.Coco} /> + // <Route exact path="/metadata/:hash/places365/" component={Metadata.Places365} /> + // <Route exact path="/metadata/:hash/sugarcube/" component={Metadata.Sugarcube} /> + // <Route exact path="/metadata/:hash/" component={Metadata.Summary} /> + // <Route exact path="/metadata/" render={() => <div className='notFound'><h4>NOT FOUND</h4></div>} /> + // <Route exact path="/search/" component={Search.Container} /> + // <Route exact path="/search/keyframe/:verified/:hash/:frame/" component={Search.Container} /> + // <Route exact path="/search/keyframe/:hash/:frame/" component={Search.Container} /> + // <Route exact path="/search/browse/:hash/" component={Search.Browse} /> + // <Route exact path="/search/random/" component={Search.Random} /> + // <Route exact path="/search/review/" component={Review.Saved} /> + // </Switch> diff --git a/scraper/client/common/activeLink.component.js b/scraper/client/common/activeLink.component.js new file mode 100644 index 00000000..59f63881 --- /dev/null +++ b/scraper/client/common/activeLink.component.js @@ -0,0 +1,16 @@ +import React from 'react' +import { NavLink } from 'react-router-dom' + +export default function ActiveLink({ + to, + className = 'navlink', + children +}) { + return ( + <span className={className}> + <NavLink to={to}> + {children} + </NavLink> + </span> + ) +} diff --git a/scraper/client/common/common.css b/scraper/client/common/common.css new file mode 100644 index 00000000..4b939df0 --- /dev/null +++ b/scraper/client/common/common.css @@ -0,0 +1,347 @@ +/* css boilerplate */ + +* { box-sizing: border-box; } +html,body { + margin: 0; padding: 0; + width: 100%; height: 100%; +} +body { + font-family: Helvetica, sans-serif; + font-weight: 300; +} + +h1 { + +} +h2 { + font-weight: normal; + margin: 10px 0; + padding: 3px; + font-size: 24px; +} +h3 { + font-weight: normal; + margin: 10px 0 0 0; + padding: 3px; + font-size: 18px; +} +h4 { + font-weight: 300; + font-size: 12px; + letter-spacing: 2px; + color: #888; + text-transform: uppercase; + margin: 5px 10px; + margin-top: 20px; +} +h4:first-child { + margin-top: 10px; +} + +.app { + width: 100%; + height: 100%; + display: flex; + flex-direction: row; + align-items: flex-start; + justify-content: flex-start; +} + +/* header stuff */ + +header { + width: 100%; + background: #11f; + color: white; + align-items: stretch; + display: flex; + flex-wrap: wrap; + justify-content: space-between; + z-index: 3; +} +header > section { + justify-content: flex-start; + align-items: center; + display: flex; + flex: 1 0; + font-weight: bold; +} +header > section:last-of-type { + justify-content: flex-end; +} + +/* sidebar / body columns */ + +.sidebar { + display: flex; + flex-direction: column; + justify-content: flex-start; + align-items: flex-start; + height: 100%; + float: left; + width: 200px; + flex: 0 0 200px; + padding: 10px; + margin-right: 10px; +} +.sidebar a { + display: block; + padding: 10px 10px; + text-decoration: none; + color: #444; +} +.sidebar a.active { + font-weight: bold; + color: #222; +} +.body { + display: flex; + flex-direction: column; + align-items: flex-start; + justify-content: flex-start; + flex-grow: 1; +} +.body > div { + padding-bottom: 40px; +} + +/* buttons / forms */ + +.btn:focus, .btn:hover { + background: #f1f1fc; + color: #4b48d6 !important; + text-decoration: none; +} +.btn { + -webkit-appearance: none; + -moz-appearance: none; + appearance: none; + background: #fff; + border: .05rem solid; + border-radius: 2px; + margin-right: 5px; + color: #11f; + cursor: pointer; + display: inline-block; + font-size: .8rem; + height: 1.8rem; + line-height: 1rem; + outline: none; + padding: .35rem .4rem; + text-align: center; + text-decoration: none; + -webkit-transition: all .2s ease; + -o-transition: all .2s ease; + transition: all .2s ease; + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; + vertical-align: middle; + white-space: nowrap; +} +.btn.reset, +.btn.panic { + color: #b00; +} +.btn.btn-primary { + background: #11f; + border-color: #11f; + color: white; +} +.btn[disabled] { + color: #bbb !important; + border-color: #bbb !important; + background: white !important; + cursor: default; +} +.btn.btn-primary:focus, +.btn.btn-primary:hover { + background: #0808ee; + color: white !important; +} +.row .btn { + margin: 0 5px 0 0; +} +input[type=text] { + border: 1px solid #888; + padding: 4px; + font-size: 15px; +} + + +/* tables on metadata pages */ + +table { + border: 0; + margin: 0; + padding: 0; + border-spacing: 0; +} +.tableObject td, +.tableObject th { + padding: 3px; + vertical-align: top; +} +.tableObject hr { + width: 100%; + color: transparent; + border: 0; + border-bottom: 1px solid #bbb; + align: left; + margin: 3px 0; + padding: 0; +} +.tableObject th, +.tableTuples th { + min-width: 145px; + text-align: left; + text-transform: capitalize; + padding: 3px; + padding-right: 10px; + font-weight: 300; + color: #333; +} +.tableTuples td { + text-align: right; + padding: 3px; +} +.tableObject td { + font-weight: normal; + color: #000; +} +.tableObject .tableObject { + border: 1px solid #ddd; +} +.tableArray { + border: 1px solid #ddd; + border-spacing: 0; +} +.tableArray td { + border-bottom: 1px solid #ddd; +} +.gray { + font-size: 12px; + color: #888; + display: block; +} +.sha256.heading { + margin: 20px 0 0px; +} +.gray span { + padding-right: 5px; +} +.gray { + margin-bottom: 10px; +} +.gray a { + color: #666; +} + +.verified { + color: #080; + font-weight: bold; +} +.unverified { + color: #f00; + font-weight: 300; +} + +.loading, .error { + font-weight: normal; + margin: 10px 0; + padding: 3px; + font-size: 24px; +} + +.title { + text-transform: capitalize; +} +.rect { + position: absolute; +} +.rect { border: 1px solid rgba(0,0,255); background-color: rgba(0,0,255,0.1); } + +/* videos / video preloader */ + +video { + max-width: 640px; + margin: 10px 0; +} +.video { + margin: 0 0 10px 0; +} +.video .bg { + cursor: pointer; + position: relative; + background-size: cover; +} +.video .play { + position: absolute; + top: 50%; + left: 50%; + transform: translate3d(-50%, -50%, 0); + width: 20%; + height: 20%; + background-image: url(/search/static/img/play.png); + background-position: center center; + background-size: contain; + background-repeat: no-repeat; +} +.desktop .video .play:hover { + -webkit-filter: invert(60%) sepia(100%) saturate(500%) hue-rotate(160deg); +} + +/* spectre.css loader */ + +.loaderWrapper { + display: inline-block; + position: relative; + width: .8rem; + height: .8rem; + padding: 10px; +} +.loader { + color: transparent !important; + min-height: .8rem; + pointer-events: none; + position: relative; +} + +.loader::after { + animation: loader 500ms infinite linear; + border: .1rem solid #5755d9; + border-radius: 50%; + border-right-color: transparent; + border-top-color: transparent; + content: ""; + display: block; + height: .8rem; + left: 50%; + margin-left: -.4rem; + margin-top: -.4rem; + position: absolute; + top: 50%; + width: .8rem; + z-index: 1; +} + +.loader.loader-lg { + min-height: 2rem; +} + +.loader.loader-lg::after { + height: 1.6rem; + margin-left: -.8rem; + margin-top: -.8rem; + width: 1.6rem; +} + +@keyframes loader { + 0% { + transform: rotate(0deg); + } + 100% { + transform: rotate(360deg); + } +}
\ No newline at end of file diff --git a/scraper/client/common/footer.component.js b/scraper/client/common/footer.component.js new file mode 100644 index 00000000..7c82b44b --- /dev/null +++ b/scraper/client/common/footer.component.js @@ -0,0 +1,10 @@ +import React from 'react' +import { Link } from 'react-router-dom' +import { connect } from 'react-redux' + +export default function Footer(props) { + return ( + <footer> + </footer> + ); +} diff --git a/scraper/client/common/gate.component.js b/scraper/client/common/gate.component.js new file mode 100644 index 00000000..9bf9287b --- /dev/null +++ b/scraper/client/common/gate.component.js @@ -0,0 +1,21 @@ +import React from 'react' +import { connect } from 'react-redux' + +function Gate(props) { + const { app, tag, View } = props + const data = app[tag] + if (!data) return null + if (data === 'loading') { + return <div className='tableObject loading'>{tag}{': Loading'}</div> + } + if (data.err) { + return <div className='tableObject error'>{tag}{' Error: '}{data.err}</div> + } + return <View data={data} {...props} /> +} + +const mapStateToProps = state => ({ + app: state.metadata +}) + +export default connect(mapStateToProps)(Gate) diff --git a/scraper/client/common/header.component.js b/scraper/client/common/header.component.js new file mode 100644 index 00000000..650df5fc --- /dev/null +++ b/scraper/client/common/header.component.js @@ -0,0 +1,19 @@ +import React, { Component } from 'react' +import { NavLink } from 'react-router-dom' +import { connect } from 'react-redux' + +class Header extends Component { + componentDidMount(){ + + } + render() { + return ( + ) + } +} + +const mapStateToProps = state => ({ + // hash: state.metadata.hash, +}) + +export default connect(mapStateToProps)(Sidebar) diff --git a/scraper/client/common/index.js b/scraper/client/common/index.js new file mode 100644 index 00000000..7f4d9870 --- /dev/null +++ b/scraper/client/common/index.js @@ -0,0 +1,22 @@ +import ActiveLink from './activeLink.component' +import Header from './header.component' +import Footer from './footer.component' +import Loader from './loader.component' +import Sidebar from './sidebar.component' +import Gate from './gate.component' +import { TableObject, TableArray, TableTuples, TableRow, TableCell } from './table.component' +import './common.css' + +export { + Header, + Footer, + Sidebar, + Loader, + Gate, + TableObject, + TableArray, + TableTuples, + TableRow, + TableCell, + ActiveLink, +} diff --git a/scraper/client/common/loader.component.js b/scraper/client/common/loader.component.js new file mode 100644 index 00000000..6795424b --- /dev/null +++ b/scraper/client/common/loader.component.js @@ -0,0 +1,10 @@ +import React, { Component } from 'react' + +export default function Loader() { + return ( + <div className='loaderWrapper'> + <div className='loader'> + </div> + </div> + ) +}
\ No newline at end of file diff --git a/scraper/client/common/sidebar.component.js b/scraper/client/common/sidebar.component.js new file mode 100644 index 00000000..f0dbe750 --- /dev/null +++ b/scraper/client/common/sidebar.component.js @@ -0,0 +1,25 @@ +import React, { Component } from 'react' +import { NavLink } from 'react-router-dom' +import { connect } from 'react-redux' + +class Sidebar extends Component { + render() { + const { hash } = this.props + if (!hash) { + return ( + <div className="sidebar"> + </div> + ) + } + return ( + <div className="sidebar"> + </div> + ) + } +} + +const mapStateToProps = state => ({ + hash: state.metadata.hash, +}) + +export default connect(mapStateToProps)(Sidebar) diff --git a/scraper/client/common/table.component.js b/scraper/client/common/table.component.js new file mode 100644 index 00000000..76a1d57c --- /dev/null +++ b/scraper/client/common/table.component.js @@ -0,0 +1,121 @@ +import React from 'react' + +import { formatName } from '../util' + +const __HR__ = '__HR__' + +export function TableObject({ tag, object, order, summary }) { + if (!object) return null + if (object === 'loading') { + return <div className='tableObject loading'>{tag}{': Loading'}</div> + } + if (object.err) { + return <div className='tableObject error'>{tag}{' Error: '}{object.err}</div> + } + let objects = Object.keys(object) + if (order) { + const grouped = objects.reduce((a, b) => { + const index = order.indexOf(b) + if (index !== -1) { + a.order.push([index, b]) + } else { + a.alpha.push(b) + } + return a + }, { order: [], alpha: [] }) + objects = grouped.order + .sort((a, b) => a[0] - b[0]) + .map(([i, s]) => s) + if (!summary) { + objects = objects + // .concat([__HR__]) + .concat(grouped.alpha.sort()) + } + } else { + objects = objects.sort() + } + return ( + <div> + {tag && <h3>{tag}</h3>} + <table className={'tableObject ' + tag}> + <tbody> + {objects.map((key, i) => ( + <TableRow key={key + '_' + i} name={key} value={object[key]} /> + ))} + </tbody> + </table> + </div> + ) +} + +export function TableArray({ tag, list }) { + if (!list) return null + return ( + <div> + {tag && <h3>{tag}</h3>} + <table className={'tableArray ' + tag}> + <tbody> + {list.map((value, i) => ( + <tr key={tag + '_' + i}> + <TableCell value={value} /> + </tr> + ))} + </tbody> + </table> + </div> + ) +} + +export function TableTuples({ tag, list }) { + if (!list) return null + return ( + <div> + {tag && <h3>{tag}</h3>} + <table className={'tableTuples ' + tag}> + <tbody> + {list.map(([key, ...values], i) => ( + <tr key={tag + '_' + i}> + <th>{formatName(key)}</th> + {values.map((value, j) => ( + <TableCell key={i + '_' + j} value={value} /> + ))} + </tr> + ))} + </tbody> + </table> + </div> + ) +} + +export function TableRow({ name, value }) { + if (name === __HR__) { + return ( + <tr> + <th className='tr'> + <hr /> + </th> + </tr> + ) + } + return ( + <tr> + <th>{formatName(name)}</th> + <TableCell name={name} value={value} /> + </tr> + ) +} + +export function TableCell({ value }) { + if (value && typeof value === 'object') { + if (value._raw) { + value = value.value + } else if (value.length) { + value = <TableArray nested tag={''} list={value} /> + } else { + value = <TableObject nested tag={''} object={value} /> + } + } + return ( + <td>{value}</td> + ) +} diff --git a/scraper/client/index.js b/scraper/client/index.js new file mode 100644 index 00000000..eddc5fb2 --- /dev/null +++ b/scraper/client/index.js @@ -0,0 +1,19 @@ +import React from 'react' +import ReactDOM from 'react-dom' +import { AppContainer } from 'react-hot-loader' +import { Provider } from 'react-redux' + +import App from './app' + +import { store, history } from './store' + +const container = document.createElement('div') +document.body.appendChild(container) + +ReactDOM.render( + <AppContainer> + <Provider store={store}> + <App history={history} /> + </Provider> + </AppContainer>, container +) diff --git a/scraper/client/search/browse.component.js b/scraper/client/search/browse.component.js new file mode 100644 index 00000000..e9ddb04e --- /dev/null +++ b/scraper/client/search/browse.component.js @@ -0,0 +1,77 @@ +import React, { Component } from 'react' +import { Link, withRouter } from 'react-router-dom' +import { bindActionCreators } from 'redux' +import { connect } from 'react-redux' + +import { Loader, Keyframes, Video } from '../common' +// import { Coco } from '../metadata' +import * as searchActions from './search.actions' +import * as metadataActions from '../metadata/metadata.actions' +import SearchMeta from './search.meta' + +class Browse extends Component { + componentDidMount() { + this.browse() + } + + componentDidUpdate(prevProps) { + if (prevProps.match.params !== this.props.match.params) { + this.browse() + } + } + + browse() { + const { hash } = this.props.match.params + if (hash) { + this.props.searchActions.browse(hash) + } + if (hash) { + this.props.metadataActions.fetchMetadata(hash) + } + } + + render() { + const { browse, options } = this.props + console.log('browse', browse) + + if (!browse || browse.reset || browse.loading) { + return <div className="browseComponent column"><h3>Loading keyframes...</h3><Loader /></div> + } + return ( + <div className="browseComponent column"> + <h3>Video Preview</h3> + <Video size={'md'} /> + <SearchMeta query={browse} sugarcube /> + <div className='row buttons'> + <Link + to={'/metadata/' + browse.hash} + className='btn' + > + View Full Metadata + </Link> + </div> + <h3>Keyframes</h3> + <Keyframes + frames={browse.frames} + showHash + showTimestamp + showSearchButton + showSaveButton + /> + </div> + ) + } +} + +const mapStateToProps = state => ({ + browse: state.search.browse, + options: state.search.options, + metadata: state.metadata, +}) + +const mapDispatchToProps = dispatch => ({ + searchActions: bindActionCreators({ ...searchActions }, dispatch), + metadataActions: bindActionCreators({ ...metadataActions }, dispatch), +}) + +export default withRouter(connect(mapStateToProps, mapDispatchToProps)(Browse)) diff --git a/scraper/client/search/index.js b/scraper/client/search/index.js new file mode 100644 index 00000000..99c2b74b --- /dev/null +++ b/scraper/client/search/index.js @@ -0,0 +1,17 @@ +import Menu from './search.menu' +import Container from './search.container' +import Meta from './search.meta' +import Query from './search.query' +import Results from './search.results' +import Browse from './browse.component' + +import './search.css' + +export { + Menu, + Container, + Meta, + Query, + Results, + Browse, +} diff --git a/scraper/client/search/panicButton.component.js b/scraper/client/search/panicButton.component.js new file mode 100644 index 00000000..a12c817b --- /dev/null +++ b/scraper/client/search/panicButton.component.js @@ -0,0 +1,49 @@ +import React, { Component } from 'react' +import { withRouter } from 'react-router-dom' +import { bindActionCreators } from 'redux' +import { connect } from 'react-redux' + +import * as actions from './search.actions' + +class PanicButton extends Component { + constructor() { + super() + this.keydown = this.keydown.bind(this) + } + + componentDidMount() { + document.addEventListener('keydown', this.keydown) + } + + componentWillUnmount() { + document.removeEventListener('keydown', this.keydown) + } + + keydown(e) { + if (e.keyCode === 27) { + this.panic() + } + } + + panic() { + this.props.actions.panic() + this.props.history.push('/search/') + } + + render() { + return ( + <button className='btn panic' onClick={() => this.panic()}> + <span>⚠</span> Panic + </button> + ) + } +} + +const mapStateToProps = state => ({ +}) + +const mapDispatchToProps = dispatch => ({ + actions: bindActionCreators({ panic: actions.panic }, dispatch) +}) + +export default withRouter(connect(mapStateToProps, mapDispatchToProps)(PanicButton)) diff --git a/scraper/client/search/search.actions.js b/scraper/client/search/search.actions.js new file mode 100644 index 00000000..95cea433 --- /dev/null +++ b/scraper/client/search/search.actions.js @@ -0,0 +1,162 @@ +// import fetchJsonp from 'fetch-jsonp' +import * as types from '../types' +// import { hashPath } from '../util' +import { store, history } from '../store' +import { post, pad, verify, preloadImage } from '../util' +import querystring from 'query-string' + +// urls + +const url = { + upload: () => process.env.API_HOST + '/search/api/upload', + search: () => process.env.API_HOST + '/search/api/fetch', + searchByVerifiedFrame: (verified, hash, frame) => process.env.API_HOST + '/search/api/search/' + verified + '/' + hash + '/' + pad(frame, 6), + searchByFrame: (hash, frame) => process.env.API_HOST + '/search/api/search/' + hash + '/' + pad(frame, 6), + browse: hash => process.env.API_HOST + '/search/api/list/' + hash, + random: () => process.env.API_HOST + '/search/api/random', + check: () => process.env.API_HOST + '/api/images/import/search', +} +export const publicUrl = { + browse: hash => '/search/browse/' + hash, + searchByVerifiedFrame: (verified, hash, frame) => '/search/keyframe/' + verify(verified) + '/' + hash + '/' + pad(frame, 6), + searchByFrame: (hash, frame) => '/search/keyframe/' + hash + '/' + pad(frame, 6), + review: () => '/search/review/' +} + +// standard loading events + +const loading = (tag, offset) => ({ + type: types.search.loading, + tag, + offset +}) +const loaded = (tag, data, offset = 0) => ({ + type: types.search.loaded, + tag, + data, + offset +}) +const error = (tag, err) => ({ + type: types.search.error, + tag, + err +}) + +// search UI functions + +export const panic = () => dispatch => { + history.push('/search/') + dispatch({ type: types.search.panic }) +} +export const updateOptions = opt => dispatch => { + dispatch({ type: types.search.update_options, opt }) +} + +// API functions + +export const upload = (file, query) => dispatch => { + const { options } = store.getState().search + const tag = 'query' + const fd = new FormData() + fd.append('query_img', file) + fd.append('limit', options.perPage) + if (!query) { + dispatch(loading(tag)) + } + post(url.upload(), fd) + .then(data => { + if (query) { + const { timing } = data.query + data.query = { + ...query, + timing, + } + let qs = {} + if (data.query.crop) { + let { x, y, w, h } = data.query.crop + qs.crop = [x, y, w, h].map(n => parseInt(n, 10)).join(',') + } + if (query.url && !query.hash) { + qs.url = query.url + } + // history.push(window.location.pathname + '#' + querystring.stringify(qs)) + // window.location.hash = querystring.stringify(qs) + } else if (data.query.url && !window.location.search.match(data.query.url)) { + history.push('/search/?url=' + data.query.url) + } + dispatch(loaded(tag, data)) + }) + .catch(err => dispatch(error(tag, err))) +} +export const searchByVerifiedFrame = (verified, hash, frame, offset = 0) => dispatch => { + const { options } = store.getState().search + const tag = 'query' + dispatch(loading(tag, offset)) + const qs = querystring.stringify({ limit: options.perPage, offset }) + preloadImage({ verified, hash, frame }) + fetch(url.searchByVerifiedFrame(verified, hash, frame) + '?' + qs, { + method: 'GET', + mode: 'cors', + }) + .then(data => data.json()) + .then(data => dispatch(loaded(tag, data, offset))) + .catch(err => dispatch(error(tag, err))) +} +export const searchByFrame = (hash, frame, offset = 0) => dispatch => { + const { options } = store.getState().search + const tag = 'query' + dispatch(loading(tag, offset)) + const qs = querystring.stringify({ limit: options.perPage, offset }) + preloadImage({ verified: false, hash, frame }) + fetch(url.searchByFrame(hash, frame) + '?' + qs, { + method: 'GET', + mode: 'cors', + }) + .then(data => data.json()) + .then(data => dispatch(loaded(tag, data, offset))) + .catch(err => dispatch(error(tag, err))) +} +export const search = (uri, offset = 0) => dispatch => { + const { options } = store.getState().search + const tag = 'query' + dispatch(loading(tag, offset)) + const qs = querystring.stringify({ url: uri, limit: options.perPage, offset }) + if (uri.indexOf('static') === 0) { + preloadImage({ uri }) + } + fetch(url.search(uri) + '?' + qs, { + method: 'GET', + mode: 'cors', + }) + .then(data => data.json()) + .then(data => dispatch(loaded(tag, data, offset))) + .catch(err => dispatch(error(tag, err))) +} +export const browse = hash => dispatch => { + const tag = 'browse' + dispatch(loading(tag)) + fetch(url[tag](hash), { + method: 'GET', + mode: 'cors', + }) + .then(data => data.json()) + .then(data => dispatch(loaded(tag, data))) + .catch(err => dispatch(error(tag, err))) +} +export const random = () => dispatch => { + const { options } = store.getState().search + const qs = querystring.stringify({ limit: options.perPage }) + const tag = 'query' + dispatch(loading(tag)) + fetch(url.random() + '?' + qs, { + method: 'GET', + mode: 'cors', + }) + .then(data => data.json()) + .then(data => { + dispatch(loaded(tag, data)) + history.push(publicUrl.searchByVerifiedFrame(data.query.verified, data.query.hash, data.query.frame)) + // window.history.pushState(null, 'VSearch: Results', publicUrl.searchByVerifiedFrame(data.query.verified, data.query.hash, data.query.frame)) + }) + .catch(err => dispatch(error(tag, err))) +} diff --git a/scraper/client/search/search.container.js b/scraper/client/search/search.container.js new file mode 100644 index 00000000..965d7c8e --- /dev/null +++ b/scraper/client/search/search.container.js @@ -0,0 +1,101 @@ +import React, { Component } from 'react' +import { withRouter } from 'react-router-dom' +import { bindActionCreators } from 'redux' +import { connect } from 'react-redux' +import * as querystring from 'querystring' + +import * as searchActions from './search.actions' +import * as metadataActions from '../metadata/metadata.actions' + +import SearchQuery from './search.query' +import SearchResults from './search.results' +import SearchSafety from './search.safety' + +class SearchContainer extends Component { + componentDidMount() { + const qs = querystring.parse(this.props.location.search.substr(1)) + if (qs && qs.url) { + this.props.searchActions.search(qs.url) + } else { + this.searchByHash() + } + } + + componentDidUpdate(prevProps) { + if (prevProps.match.params !== this.props.match.params && JSON.stringify(this.props.match.params) !== JSON.stringify(prevProps.match.params)) { + this.searchByHash() + } + // const qsOld = querystring.parse(prevProps.location.search.substr(1)) + // const qsNew = querystring.parse(this.props.location.search.substr(1)) + // if (qsOld && qsNew && qsNew.url && qsNew.url !== qsOld.url) { + // this.props.actions.search(qsNew.url) + // } + } + + searchByHash(offset = 0) { + const { verified, hash, frame } = this.props.match.params + if (verified && hash && frame) { + this.props.searchActions.searchByVerifiedFrame(verified, hash, frame, offset) + } else if (hash && frame) { + this.props.searchActions.searchByFrame(hash, frame, offset) + } + if (hash && !offset) { + this.props.metadataActions.fetchMetadata(hash) + } + } + + searchByOffset() { + const offset = this.props.query.results.length + const qs = querystring.parse(this.props.location.search.substr(1)) + if (qs && qs.url) { + this.props.searchActions.search(qs.url, offset) + } + else { + this.searchByHash(offset) + } + } + + render() { + const { query, results, loadingMore } = this.props.query + const options = this.props.options + // console.log('search container', query, results, loadingMore) + let showLoadMore = true + if (!query || query.reset || query.loading || !results || !results.length) { + showLoadMore = false + } + let isWide = (results && results.length > Math.min(options.perPage, 30)) + let isMoreLoaded = (results && results.length > options.perPage) + return ( + <div className='searchContainer'> + <SearchQuery /> + <SearchResults /> + {showLoadMore + ? !loadingMore + ? <button + onClick={() => this.searchByOffset()} + className={isWide ? 'btn loadMore wide' : 'btn loadMore'} + > + Load more + </button> + : <div className='loadingMore'>{'Loading more results...'}</div> + : <div> + </div> + } + {!isMoreLoaded && <SearchSafety />} + </div> + ) + } +} + +const mapStateToProps = state => ({ + query: state.search.query, + options: state.search.options, + metadata: state.metadata, +}) + +const mapDispatchToProps = dispatch => ({ + searchActions: bindActionCreators({ ...searchActions }, dispatch), + metadataActions: bindActionCreators({ ...metadataActions }, dispatch), +}) + +export default withRouter(connect(mapStateToProps, mapDispatchToProps)(SearchContainer)) diff --git a/scraper/client/search/search.css b/scraper/client/search/search.css new file mode 100644 index 00000000..280e53fe --- /dev/null +++ b/scraper/client/search/search.css @@ -0,0 +1,230 @@ +.btn span { + font-size: large; +} +.row { + display: flex; + flex-direction: row; +} +.column { + display: flex; + flex-direction: column; +} + +.searchContainer h3 { + padding: 0; + margin-top: 0; + margin-bottom: 5px; + margin-left: 3px; +} +.searchContainer h4 { + margin-left: 0; + width: 100%; +} +.searchContainer .subtitle { + display: block; + margin-left: 3px; + margin-bottom: 10px; +} +.searchForm { + display: flex; + justify-content: space-between; + align-items: center; + width: 100%; + padding: 20px; + background: #eee; +} +.searchForm .row { + align-items: center; +} + +.searchMeta { + display: flex; + flex-direction: column; + font-size: 14px; + line-height: 18px; + padding: 0; +} +.searchMeta span { + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + max-width: calc(100vw - 23px - 640px - 30px); +} + +.keyframe .thumbnail { + position: relative; + cursor: pointer; +} +.keyframe .searchButtons { + position: absolute; + bottom: 0; left: 0; + padding: 0 5px 15px 5px; + width: 100%; + text-align: center; + opacity: 0; + transition: all 0.2s; +} +.desktop .keyframe .thumbnail:hover .searchButtons, +.mobile .keyframe .searchButtons { + opacity: 1; +} +.keyframe .searchButtons .btn { + margin-right: 0; + height: auto; + padding: 0.15rem 0.3rem; +} +.keyframe a { + text-decoration: none; +} + +.body > div.searchForm { + padding-bottom: 20px; +} +.upload { + position: relative; + cursor: pointer; +} +.upload .btn { + pointer-events: none; + cursor: pointer; +} +.upload input { + position: absolute; + top: 0; left: 0; + width: 100%; height: 100%; + opacity: 0; + cursor: pointer; +} + +.reviewSaved, +.browseComponent, +.searchQuery { + margin: 0px 10px; + padding: 13px; +} +.searchQuery img { + cursor: crosshair; + user-select: none; + max-width: 640px; + max-height: 480px; +} +.searchContainer .searchQuery h3 { + margin-left: 0; + margin-bottom: 10px; +} + +.searchBox { + min-width: 640px; + margin: 0 10px 0 0; + background-color: #eee; + position: relative; +} +.searchBox img { + display: block; +} +.searchBox .box { + position: absolute; + cursor: crosshair; + border: 1px solid #11f; + background-color: rgba(16,16,255,0.1); +} + +.searchResults { + margin: 0 20px 20px 20px; + display: flex; + flex-direction: row; + flex-wrap: wrap; +} +.searchResultsHeading { + width: 100%; +} +.searchOptions .row { + font-size: 12px; + margin-left: 10px; +} +.searchOptions input { + font-size: 12px; + margin-right: 5px; + font-family: Helvetica, sans-serif; +} +.searchOptions input[type=text], +.searchOptions input[type=number] { + width: 30px; + text-align: right; +} +.keyframeGroup { + max-width: 650px; + display: flex; + flex-direction: row; + flex-wrap: wrap; + align-items: flex-start; + align-content: flex-start; + justify-content: flex-start; +} +.keyframeGroup h4 a { + color: #888; + text-decoration: none +} +.keyframeGroup h4 a:hover { + text-decoration: underline +} + +/* load more button that gets bigger */ + +.loadMore { + width: 400px; + margin: 20px; + height: 40px; + transition: all; +} +.loadMore.wide { + width: calc(100% - 40px); + margin: 20px; + height: 100px; +} +.loadingMore { + margin: 20px 20px 200px 20px; +} + +/* health and safety warning */ + +.safety div { + display: inline-block; + margin: 20px 20px; + padding: 10px; + background: #fff8e8; + color: #111; + box-shadow: 0 1px 2px rgba(0,0,0,0.2); + font-size: 13px; + line-height: 1.4; +} +.safety ul { + margin: 0; + padding: 0 21px; +} +.safety li { + padding: 1px 0 0 0; +} +.safety h4 { + margin-top: 5px; +} + +/* browser section */ + +.browseComponent h3 { + margin-bottom: 10px; +} +.browseComponent .buttons { + margin-top: 10px; +} + +/* disable twiddle button on input[type=number] */ + +input::-webkit-outer-spin-button, +input::-webkit-inner-spin-button { + -webkit-appearance: none; + margin: 0; +} +input[type='number'] { + -moz-appearance:textfield; +} diff --git a/scraper/client/search/search.menu.js b/scraper/client/search/search.menu.js new file mode 100644 index 00000000..f5f9423e --- /dev/null +++ b/scraper/client/search/search.menu.js @@ -0,0 +1,96 @@ +import React, { Component } from 'react' +import { Link } from 'react-router-dom' +import { bindActionCreators } from 'redux' +import { connect } from 'react-redux' + +import * as actions from './search.actions' +import PanicButton from './panicButton.component' + +class SearchMenu extends Component { + upload(e) { + const files = e.dataTransfer ? e.dataTransfer.files : e.target.files + let i + let file + for (i = 0; i < files.length; i++) { + file = files[i] + if (file && file.type.match('image.*')) break + } + if (!file) return + this.props.actions.upload(file) + } + + random() { + this.props.actions.random() + } + + render() { + const { savedCount, options } = this.props + return ( + <div className="searchForm row"> + <div className='row'> + <div className='upload'> + <button className='btn'><span>⤴</span> Search by Upload</button> + <input + type="file" + name="img" + accept="image/*" + onChange={this.upload.bind(this)} + required + /> + </div> + <button className='btn random' onClick={this.random.bind(this)}><span>♘</span> Random</button> + <PanicButton /> + <Link to={actions.publicUrl.review()}> + <button className='btn btn-primary'><span>⇪</span>{ + ' ' + savedCount + ' Saved Image' + (savedCount === 1 ? '' : 's') + }</button> + </Link> + </div> + + <div className='row searchOptions'> + <select + className='form-select' + onChange={e => this.props.actions.updateOptions({ thumbnailSize: e.target.value })} + value={options.thumbnailSize} + > + <option value='th'>Thumbnail</option> + <option value='sm'>Small</option> + <option value='md'>Medium</option> + <option value='lg'>Large</option> + </select> + <label className='row'> + <input + type='checkbox' + checked={options.groupByHash} + onChange={e => this.props.actions.updateOptions({ groupByHash: e.target.checked })} + /> + {' Group by hash'} + </label> + <label className='row'> + <input + type='number' + value={options.perPage} + className='perPage' + min={1} + max={100} + onChange={e => this.props.actions.updateOptions({ perPage: e.target.value })} + onBlur={() => window.location.reload()} + /> + {' per page'} + </label> + </div> + </div> + ) + } +} + +const mapStateToProps = state => ({ + options: state.search.options, + savedCount: state.review.count, +}) + +const mapDispatchToProps = dispatch => ({ + actions: bindActionCreators({ ...actions }, dispatch) +}) + +export default connect(mapStateToProps, mapDispatchToProps)(SearchMenu) diff --git a/scraper/client/search/search.meta.js b/scraper/client/search/search.meta.js new file mode 100644 index 00000000..b4eaeaad --- /dev/null +++ b/scraper/client/search/search.meta.js @@ -0,0 +1,68 @@ +import React, { Component } from 'react' +import { Link } from 'react-router-dom' +import { bindActionCreators } from 'redux' +import { connect } from 'react-redux' +import { format } from 'date-fns' + +import { timestamp } from '../util' +import * as searchActions from './search.actions' + +class SearchMeta extends Component { + render() { + const { query, metadata, sugarcube } = this.props + if (!query || !metadata || !metadata.mediainfo || metadata.metadata === 'loading') { + return <div className='searchMeta'></div> + } + const sugarcubeId = metadata.mediainfo.sugarcube_id + const { video } = metadata.mediainfo.metadata.mediainfo + const { x, y, w, h } = query.crop || {} + return ( + <div className='searchMeta'> + {'verified' in query && + <span className={query.verified ? 'verified' : 'unverified'}> + {query.verified ? 'verified' : 'unverified'} + </span> + } + {query.hash && + <span> + {'sha256: '} + <Link className="sha256" to={searchActions.publicUrl.browse(query.hash)}>{query.hash}</Link> + </span> + } + {query.frame && + <span> + {'Frame: '} + {timestamp(query.frame, video.frame_rate)} + {' / '} + {timestamp(video.duration / 1000, 1)} + </span> + } + {query.crop && + <span> + {'Crop: '}{parseInt(w, 10) + 'x' + parseInt(h, 10) + ' @ (' + parseInt(x, 10) + ', ' + parseInt(y, 10) + ')'} + </span> + } + {!!(video && video.encoded_date) && + <span> + {'Date: '}{format(new Date(video.encoded_date), "DD-MMM-YYYY")} + </span> + } + {!!(sugarcube && sugarcubeId) && + <span> + sugarcube: {sugarcubeId} + </span> + } + </div> + ) + } +} + +const mapStateToProps = state => ({ + metadata: state.metadata, +}) + +const mapDispatchToProps = dispatch => ({ + actions: bindActionCreators({ ...searchActions }, dispatch), +}) + +export default connect(mapStateToProps, mapDispatchToProps)(SearchMeta) diff --git a/scraper/client/search/search.query.js b/scraper/client/search/search.query.js new file mode 100644 index 00000000..276f1943 --- /dev/null +++ b/scraper/client/search/search.query.js @@ -0,0 +1,227 @@ +import React, { Component } from 'react' +import { Link } from 'react-router-dom' +import { bindActionCreators } from 'redux' +import { connect } from 'react-redux' +import toBlob from 'data-uri-to-blob' + +import { clamp, px } from '../util' +import { Loader } from '../common' + +import * as searchActions from './search.actions' +import SearchMeta from './search.meta' + +const defaultState = { + dragging: false, + draggingBox: false, + bounds: null, + mouseX: 0, + mouseY: 0, + box: { + x: 0, + y: 0, + w: 0, + h: 0, + } +} + +class SearchQuery extends Component { + state = { + ...defaultState + } + + constructor() { + super() + this.handleMouseDown = this.handleMouseDown.bind(this) + this.handleMouseDownOnBox = this.handleMouseDownOnBox.bind(this) + this.handleMouseMove = this.handleMouseMove.bind(this) + this.handleMouseUp = this.handleMouseUp.bind(this) + } + + componentDidMount() { + document.body.addEventListener('mousemove', this.handleMouseMove) + document.body.addEventListener('mouseup', this.handleMouseUp) + } + + componentDidUpdate(prevProps) { + // console.log(this.props.query.query, !prevProps.query.query) + if (this.state.bounds && (!this.props.query.query || !prevProps.query.query || this.props.query.query.url !== prevProps.query.query.url)) { + this.setState({ ...defaultState }) + } + } + + componentWillUnmount() { + document.body.removeEventListener('mousemove', this.handleMouseMove) + document.body.removeEventListener('mouseup', this.handleMouseUp) + } + + handleMouseDown(e) { + e.preventDefault() + const bounds = this.imgRef.getBoundingClientRect() + const mouseX = e.pageX + const mouseY = e.pageY + const x = mouseX - bounds.left + const y = mouseY - bounds.top + const w = 1 + const h = 1 + this.setState({ + dragging: true, + bounds, + mouseX, + mouseY, + box: { + x, y, w, h, + } + }) + } + + handleMouseDownOnBox(e) { + const bounds = this.imgRef.getBoundingClientRect() + const mouseX = e.pageX + const mouseY = e.pageY + this.setState({ + draggingBox: true, + bounds, + mouseX, + mouseY, + initialBox: { + ...this.state.box + }, + box: { + ...this.state.box + } + }) + } + + handleMouseMove(e) { + const { + dragging, draggingBox, + bounds, mouseX, mouseY, initialBox, box + } = this.state + if (dragging) { + e.preventDefault() + let { x, y } = box + let w = clamp(e.pageX - mouseX, 0, bounds.width - x) + let h = clamp(e.pageY - mouseY, 0, bounds.height - y) + this.setState({ + box: { + x, y, w, h, + } + }) + } else if (draggingBox) { + e.preventDefault() + let { x, y, w, h } = initialBox + let dx = (e.pageX - mouseX) + let dy = (e.pageY - mouseY) + this.setState({ + box: { + x: clamp(x + dx, 0, bounds.width - w), + y: clamp(y + dy, 0, bounds.height - h), + w, + h, + } + }) + } + } + + handleMouseUp(e) { + const { actions } = this.props + const { dragging, draggingBox, bounds, box } = this.state + if (!dragging && !draggingBox) return + e.preventDefault() + const { x, y, w, h } = box + // console.log(x, y, w, h) + const img = this.imgRef + const canvas = document.createElement('canvas') + const ctx = canvas.getContext('2d') + const ratio = img.naturalWidth / bounds.width + canvas.width = w * ratio + canvas.height = h * ratio + if (w < 10 || h < 10) { + this.setState({ dragging: false, draggingBox: false, box: { x: 0, y: 0, w: 0, h: 0 } }) + return + } + this.setState({ dragging: false, draggingBox: false }) + // query_div.appendChild(canvas) + const newImage = new Image() + let loaded = false + newImage.onload = () => { + if (loaded) return + loaded = true + newImage.onload = null + ctx.drawImage( + newImage, + Math.round(x * ratio), + Math.round(y * ratio), + Math.round(w * ratio), + Math.round(h * ratio), + 0, 0, canvas.width, canvas.height + ) + const blob = toBlob(canvas.toDataURL('image/jpeg', 0.9)) + actions.upload(blob, { + ...this.props.query.query, + crop: { + x, y, w, h, + } + }) + } + // console.log(img.src) + newImage.crossOrigin = 'anonymous' + newImage.src = img.src + if (newImage.complete) { + newImage.onload() + } + } + + render() { + const { query } = this.props.query + const { box } = this.state + const { x, y, w, h } = box + if (!query) return null + if (query.loading) { + return <div className="searchQuery column"><h2>Loading results...</h2><Loader /></div> + } + let { url } = query + if (url && url.indexOf('static') === 0) { + url = '/search/' + url + } + return ( + <div className="searchQuery row"> + <div className="searchBox"> + <img + src={url} + ref={ref => this.imgRef = ref} + onMouseDown={this.handleMouseDown} + crossOrigin='anonymous' + /> + {!!w && + <div + className="box" + style={{ + left: x, + top: y, + width: w, + height: h, + }} + onMouseDown={this.handleMouseDownOnBox} + /> + } + </div> + <div> + <h3>Your Query</h3> + <SearchMeta query={query} /> + </div> + </div> + ) + } +} + +const mapStateToProps = state => ({ + query: state.search.query, + options: state.search.options, +}) + +const mapDispatchToProps = dispatch => ({ + actions: bindActionCreators({ ...searchActions }, dispatch), +}) + +export default connect(mapStateToProps, mapDispatchToProps)(SearchQuery) diff --git a/scraper/client/search/search.reducer.js b/scraper/client/search/search.reducer.js new file mode 100644 index 00000000..b9de60bd --- /dev/null +++ b/scraper/client/search/search.reducer.js @@ -0,0 +1,84 @@ +import * as types from '../types' +import session from '../session' + +const initialState = () => ({ + query: { reset: true }, + browse: { reset: true }, + options: { + thumbnailSize: session('thumbnailSize') || 'th', + perPage: parseInt(session('perPage'), 10) || 50, + groupByHash: session('groupByHash'), + } +}) + +const loadingState = { + query: { + query: { loading: true }, + results: [] + }, + loading: { + loading: true + } +} + +export default function searchReducer(state = initialState(), action) { + // console.log(action.type, action) + switch (action.type) { + case types.search.loading: + if (action.tag === 'query' && action.offset) { + return { + ...state, + query: { + ...state.query, + loadingMore: true, + } + } + } + return { + ...state, + [action.tag]: loadingState[action.tag] || loadingState.loading, + } + + case types.search.loaded: + if (action.tag === 'query' && action.offset) { + return { + ...state, + query: { + query: action.data.query, + results: [ + ...state.query.results, + ...action.data.results, + ], + loadingMore: false, + } + } + } + return { + ...state, + [action.tag]: action.data, + } + + case types.search.error: + return { + ...state, + [action.tag]: { error: action.err }, + } + + case types.search.panic: + return { + ...initialState(), + } + + case types.search.update_options: + session.setAll(action.opt) + return { + ...state, + options: { + ...action.opt, + } + } + + default: + return state + } +} diff --git a/scraper/client/search/search.results.js b/scraper/client/search/search.results.js new file mode 100644 index 00000000..8b9e0c5e --- /dev/null +++ b/scraper/client/search/search.results.js @@ -0,0 +1,49 @@ +import React, { Component } from 'react' +import { Link, withRouter } from 'react-router-dom' +import { bindActionCreators } from 'redux' +import { connect } from 'react-redux' +import * as querystring from 'querystring' + +import { Keyframes } from '../common' +import * as searchActions from './search.actions' + +function SearchResults({ query, results, options }) { + if (!query || query.reset || query.loading || !results) { + return <div></div> + } + if (!query.loading && !results.length) { + return <div className='searchResults'><h3>No results</h3></div> + } + return ( + <div className="searchResults"> + <div className='searchResultsHeading row'> + <div className='column'> + <h3>Search Results</h3> + <small className="subtitle"> + {'Searched 10,523,176 frames from 576,234 videos (took '}{query.timing.toFixed(2)}{' ms)'} + </small> + </div> + </div> + <Keyframes + frames={results} + showHash + showTimestamp={options.groupByHash} + showSearchButton + showSaveButton + groupByHash={options.groupByHash} + /> + </div> + ) +} + +const mapStateToProps = state => ({ + query: state.search.query.query, + results: state.search.query.results, + options: state.search.options, +}) + +const mapDispatchToProps = dispatch => ({ + searchActions: bindActionCreators({ ...searchActions }, dispatch), +}) + +export default withRouter(connect(mapStateToProps, mapDispatchToProps)(SearchResults)) diff --git a/scraper/client/search/search.safety.js b/scraper/client/search/search.safety.js new file mode 100644 index 00000000..b4f92664 --- /dev/null +++ b/scraper/client/search/search.safety.js @@ -0,0 +1,17 @@ +import React from 'react' + +export default function SearchWarning() { + return ( + <div className='safety'> + <div> + <h4>Safety Tips</h4> + <ul> + <li> Look away if you see something traumatic </li> + <li> Hit <tt>ESC</tt> to activate panic mode (hides all images) </li> + <li> Use thumbnails to reduce details </li> + <li> Take breaks and refresh yourself with positive imagery </li> + </ul> + </div> + </div> + ) +} diff --git a/scraper/client/session.js b/scraper/client/session.js new file mode 100644 index 00000000..2f006451 --- /dev/null +++ b/scraper/client/session.js @@ -0,0 +1,5 @@ +import Storage from 'store2' + +const session = Storage.namespace('megapixels.scraper') + +export default session diff --git a/scraper/client/store.js b/scraper/client/store.js new file mode 100644 index 00000000..08b6e801 --- /dev/null +++ b/scraper/client/store.js @@ -0,0 +1,64 @@ +import { applyMiddleware, compose, combineReducers, createStore } from 'redux' +import { connectRouter, routerMiddleware } from 'connected-react-router' +import { createBrowserHistory } from 'history' +import thunk from 'redux-thunk' +import * as types from './types' +// import { login } from './util' + +// import metadataReducer from './metadata/metadata.reducer' + +const initialState = () => ({ + options: {} +}) + +export default function apiReducer(state = initialState(), action) { + // console.log(action.type, action) + switch (action.type) { + case types.api.loading: + return { + ...state, + [action.tag]: { loading: true }, + } + + case types.api.loaded: + return { + ...state, + [action.tag]: action.data, + } + + case types.api.error: + return { + ...state, + [action.tag]: { error: action.err }, + } + + default: + return state + } +} + +const rootReducer = combineReducers({ + api: apiReducer, +}) + +function configureStore(initialState = {}, history) { + const composeEnhancers = window.__REDUX_DEVTOOLS_EXTENSION_COMPOSE__ || compose + + const store = createStore( + connectRouter(history)(rootReducer), // new root reducer with router state + initialState, + composeEnhancers( + applyMiddleware( + thunk, + routerMiddleware(history) + ), + ), + ) + + return store +} + +const history = createBrowserHistory() +const store = configureStore({}, history) + +export { store, history } diff --git a/scraper/client/types.js b/scraper/client/types.js new file mode 100644 index 00000000..8507d556 --- /dev/null +++ b/scraper/client/types.js @@ -0,0 +1,13 @@ +export const asType = (type, name) => [type, name].join('_').toUpperCase() +export const tagAsType = (type, names) => ( + names.reduce((tags, name) => { + tags[name] = asType(type, name) + return tags + }, {}) +) + +export const api = tagAsType('api', [ + 'loading', 'loaded', 'error', +]) + +export const init = '@@INIT' diff --git a/scraper/client/util.js b/scraper/client/util.js new file mode 100644 index 00000000..ef93507e --- /dev/null +++ b/scraper/client/util.js @@ -0,0 +1,149 @@ +/* Mobile check */ + +export const isiPhone = !!((navigator.userAgent.match(/iPhone/i)) || (navigator.userAgent.match(/iPod/i))) +export const isiPad = !!(navigator.userAgent.match(/iPad/i)) +export const isAndroid = !!(navigator.userAgent.match(/Android/i)) +export const isMobile = isiPhone || isiPad || isAndroid +export const isDesktop = !isMobile + +const htmlClassList = document.body.parentNode.classList +htmlClassList.add(isDesktop ? 'desktop' : 'mobile') + +/* Default image dimensions */ + +export const widths = { + th: 160, + sm: 320, + md: 640, + lg: 1280, +} + +/* Formatting functions */ + +const acronyms = 'id url cc sa fp md5 sha256'.split(' ').map(s => '_' + s) +const acronymsUpperCase = acronyms.map(s => s.toUpperCase()) + +export const formatName = s => { + acronyms.forEach((acronym, i) => s = s.replace(acronym, acronymsUpperCase[i])) + return s.replace(/_/g, ' ') +} + +// Use to pad frame numbers with zeroes +export const pad = (n, m) => { + let s = String(n || 0) + while (s.length < m) { + s = '0' + s + } + return s +} + +// Verified is 0/1 when retrieved from SQL, but 'verified' or 'unverified' when retrieved elsewhere +export const isVerified = verified => verified === 1 || verified === '1' || verified === 'verified' +export const verify = verified => isVerified(verified) ? 'verified' : 'unverified' + +export const courtesyS = (n, s) => n + ' ' + (n === 1 ? s : s + 's') + +export const padSeconds = n => n < 10 ? '0' + n : n + +export const timestamp = (n = 0, fps = 25) => { + n /= fps + let s = padSeconds(Math.round(n) % 60) + n = Math.floor(n / 60) + if (n > 60) { + return Math.floor(n / 60) + ':' + padSeconds(n % 60) + ':' + s + } + return (n % 60) + ':' + s +} + +export const percent = n => (n * 100).toFixed(1) + '%' + +export const px = (n, w) => Math.round(n * w) + 'px' + +export const clamp = (n, a, b) => n < a ? a : n < b ? n : b + +/* URLs */ + +export const hashPath = sha256 => { + if (!sha256 || sha256.length < 9) { + throw new Error('Invalid sha256') + } + return [ + sha256.slice(0, 3), + sha256.slice(3, 6), + sha256.slice(6, 9), + sha256, + ].join('/') +} + +export const imageUrl = (verified, sha256, frame, size = 'th') => [ + 'https://' + process.env.S3_HOST + '/v1/media/keyframes', + isVerified(verified) ? null : 'unverified', + hashPath(sha256), + pad(frame, 6), + size, + 'index.jpg' +].filter(s => !!s).join('/') + +export const metadataUri = (sha256, tag) => '/metadata/' + sha256 + '/' + tag + '/' +export const keyframeUri = (sha256, frame) => '/metadata/' + sha256 + '/keyframe/' + pad(frame, 6) + '/' + +export const preloadImage = opt => { + let { verified, hash, frame, url } = opt + if (hash && frame) { + url = imageUrl(verified, hash, frame, 'md') + } + const image = new Image() + let loaded = false + image.onload = () => { + if (loaded) return + loaded = true + image.onload = null + } + // console.log(img.src) + image.crossOrigin = 'anonymous' + image.src = url + if (image.complete) { + image.onload() + } +} + +/* AJAX */ + +export const get = (uri, data) => { + let headers = { + Accept: 'application/json, application/xml, text/play, text/html, *.*', + } + let opt = { + method: 'GET', + body: data, + headers, + // credentials: 'include', + } + // console.log(headers) + // headers['X-CSRFToken'] = csrftoken + return fetch(uri, opt).then(res => res.json()) +} + +export const post = (uri, data) => { + let headers + if (data instanceof FormData) { + headers = { + Accept: 'application/json, application/xml, text/play, text/html, *.*', + } + } else { + headers = { + Accept: 'application/json, application/xml, text/play, text/html, *.*', + 'Content-Type': 'application/json; charset=utf-8', + } + data = JSON.stringify(data) + } + let opt = { + method: 'POST', + body: data, + headers, + // credentials: 'include', + } + // console.log(headers) + // headers['X-CSRFToken'] = csrftoken + return fetch(uri, opt).then(res => res.json()) +} |