listing citations, known and unknown

11 files changed, 226 insertions, 9 deletions

diff --git a/scraper/client/app.js b/scraper/client/app.js
index 4bc6276c..7600fe52 100644
--- a/scraper/client/app.js
+++ b/scraper/client/app.js
@@ -15,7 +15,12 @@ export default class App extends Component {
             <div className='body'>
               <Route path="/paper/:key/" component={Paper.Manager} />
               <Switch>
+                <Route exact path="/paper/:key/" component={Paper.Random} />
+                <Route exact path="/paper/:key/citations/" component={Paper.Citations} />
+                <Route exact path="/paper/:key/unknown/" component={Paper.UnknownCitations} />
                 <Route exact path="/paper/:key/info/" component={Paper.Info} />
+                <Route exact path="/paper/:key/random/" component={Paper.Random} />
+                <Route exact path="/paper/:key/address/:sha256" component={Paper.Address} />
               </Switch>
             </div>
           </div>
@@ -25,7 +30,6 @@ export default class App extends Component {
     )
   }
 }
-
 // <Route path="/search/" component={Search.Menu} />
 // <Route path="/metadata/:hash/" component={Metadata.Heading} />
 // <Switch>
diff --git a/scraper/client/common/header.component.js b/scraper/client/common/header.component.js
index a14d925f..0cd67c76 100644
--- a/scraper/client/common/header.component.js
+++ b/scraper/client/common/header.component.js
@@ -9,8 +9,8 @@ import * as actions from '../actions'
 
 class Header extends Component {
   componentDidMount() {
-    this.props.actions.getInstitutions()
     this.props.actions.getPapers()
+    this.props.actions.getInstitutions()
   }
 
   pickPaper(e) {
diff --git a/scraper/client/common/table.component.js b/scraper/client/common/table.component.js
index f9be0669..96b62835 100644
--- a/scraper/client/common/table.component.js
+++ b/scraper/client/common/table.component.js
@@ -15,7 +15,7 @@ export function TableObject({ tag, object, order, summary }) {
   let keys = Object.keys(object)
   if (order) {
     const grouped = keys.reduce((a, b) => {
-      if (summary && !object[b].trim().length) {
+      if (summary && !(object[b].trim ? object[b].trim().length : object[b])) {
         return a
       }
       const index = order.indexOf(b)
diff --git a/scraper/client/paper/index.js b/scraper/client/paper/index.js
index c10ea011..4fad0316 100644
--- a/scraper/client/paper/index.js
+++ b/scraper/client/paper/index.js
@@ -1,5 +1,9 @@
 import Manager from './paper.manager'
 import Info from './paper.info'
+import Citations from './paper.citations'
+import UnknownCitations from './paper.unknown'
+import Random from './paper.random'
+import Address from './paper.address'
 import './paper.css'
 
 // import './search.css'
@@ -7,4 +11,8 @@ import './paper.css'
 export {
   Manager,
   Info,
+  Citations,
+  UnknownCitations,
+  Random,
+  Address,
 }
diff --git a/scraper/client/paper/paper.address.js b/scraper/client/paper/paper.address.js
new file mode 100644
index 00000000..2240388f
--- /dev/null
+++ b/scraper/client/paper/paper.address.js
@@ -0,0 +1,38 @@
+import React, { Component } from 'react'
+import { bindActionCreators } from 'redux'
+import { connect } from 'react-redux'
+
+import * as actions from '../actions'
+
+// import { Loader } from '../common'
+
+class PaperAddress extends Component {
+  componentDidMount() {
+    const { sha256 } = this.props.match.params
+    this.props.actions.getAddress(sha256)
+  }
+
+  componentDidUpdate(newProps) {
+    const { sha256: oldSha256 } = this.props.match.params
+    const { sha256 } = newProps.match.params
+    if (sha256 !== oldSha256) {
+      this.props.actions.getPaperInfo(this.props.match.params.key)
+    }
+  }
+
+  render() {
+    // if (!this.props.match.params.key) return null
+    // if (this.props.api.address.loading) return <Loader />
+    // if (!this.props.api.paperInfo.dataset) return <div className='paperInfo'>Metadata not found</div>
+    return null
+  }
+}
+
+const mapStateToProps = state => ({
+  api: state.api,
+})
+const mapDispatchToProps = dispatch => ({
+  actions: bindActionCreators({ ...actions }, dispatch),
+})
+
+export default connect(mapStateToProps, mapDispatchToProps)(PaperAddress)
diff --git a/scraper/client/paper/paper.citations.js b/scraper/client/paper/paper.citations.js
new file mode 100644
index 00000000..de423a77
--- /dev/null
+++ b/scraper/client/paper/paper.citations.js
@@ -0,0 +1,58 @@
+import React, { Component } from 'react'
+import { bindActionCreators } from 'redux'
+import { connect } from 'react-redux'
+
+import * as actions from '../actions'
+
+import { TableObject } from '../common'
+
+class PaperCitations extends Component {
+  render() {
+    const { dataset, citations } = this.props.api.paperInfo
+    if (!dataset || !citations) return null
+    console.log('rendering citations...')
+    return (
+      <div className='citations'>
+        <h2>{dataset.name_full}: Citations</h2>
+        <ul>
+        {citations.map((citation, i) => {
+          let cite = { ...citation }
+          cite.id = {
+            _raw: true,
+            value: <a href={'/api/address/' + dataset.key + '/' + citation.id}>{citation.id}</a>
+          }
+          cite.pdf = {
+            _raw: true,
+            value: cite.pdf ? <a href={cite.pdf} rel='noopener noreferrer' target="_blank">[pdf]</a> : "no pdf"
+          }
+          cite.addresses = {
+            _raw: true,
+            value: cite.addresses.map((address, j) => (
+              <div key={j}>{address.address}{', '}<span className='type'>{address.type}</span></div>
+            ))
+          }
+          return (
+            <li key={i}>
+              <TableObject
+                summary
+                object={cite}
+                tag={cite.title}
+                order={['id', 'pdf', 'year', 'addresses']}
+              />
+            </li>
+          )
+        })}
+        </ul>
+      </div>
+    )
+  }
+}
+
+const mapStateToProps = state => ({
+  api: state.api
+})
+const mapDispatchToProps = dispatch => ({
+  actions: bindActionCreators({ ...actions }, dispatch),
+})
+
+export default connect(mapStateToProps, mapDispatchToProps)(PaperCitations)
diff --git a/scraper/client/paper/paper.css b/scraper/client/paper/paper.css
index c1a775c1..cd2f8529 100644
--- a/scraper/client/paper/paper.css
+++ b/scraper/client/paper/paper.css
@@ -1,3 +1,19 @@
 .paperInfo {
   padding: 10px;
+}
+
+.citations ul {
+  list-style-type: none;
+  margin: 0;
+  padding: 0;
+}
+
+.citations li {
+  list-style-type: none;
+  margin: 0;
+  padding-bottom: 40px;
+}
+
+.type {
+  color: #888;
 }
\ No newline at end of file
diff --git a/scraper/client/paper/paper.random.js b/scraper/client/paper/paper.random.js
new file mode 100644
index 00000000..aab22172
--- /dev/null
+++ b/scraper/client/paper/paper.random.js
@@ -0,0 +1,36 @@
+import React, { Component } from 'react'
+// import { NavLink } from 'react-router-dom'
+import { bindActionCreators } from 'redux'
+import { connect } from 'react-redux'
+
+import { choice } from '../util'
+import { history } from '../store'
+
+import * as actions from '../actions'
+
+class PaperRandom extends Component {
+  componentDidUpdate() {
+    const { citations } = this.props.api.unknownCitations
+    if (!citations) return
+    const citation = choice(citations)
+    console.log(citation)
+    if (citation.id) {
+      history.push('/paper/' + this.props.match.params.key + '/address/' + citation.id)
+    }
+  }
+
+  render() {
+    return (
+      <div>Sending you to a random citation...</div>
+    )
+  }
+}
+
+const mapStateToProps = state => ({
+  api: state.api
+})
+const mapDispatchToProps = dispatch => ({
+  actions: bindActionCreators({ ...actions }, dispatch),
+})
+
+export default connect(mapStateToProps, mapDispatchToProps)(PaperRandom)
diff --git a/scraper/client/paper/paper.unknown.js b/scraper/client/paper/paper.unknown.js
new file mode 100644
index 00000000..7a20f398
--- /dev/null
+++ b/scraper/client/paper/paper.unknown.js
@@ -0,0 +1,53 @@
+import React, { Component } from 'react'
+import { bindActionCreators } from 'redux'
+import { connect } from 'react-redux'
+
+import * as actions from '../actions'
+
+import { TableObject } from '../common'
+
+class PaperUnknown extends Component {
+  render() {
+    const { dataset } = this.props.api.paperInfo
+    const { citations } = this.props.api.unknownCitations
+    if (!dataset || !citations) return null
+    console.log('rendering unknown citations...')
+    return (
+      <div className='citations'>
+        <h2>{dataset.name_full}: Unknown Citations</h2>
+        <ul>
+        {citations.map((citation, i) => {
+          let cite = { ...citation }
+          cite.id = {
+            _raw: true,
+            value: <a href={'/api/address/' + dataset.key + '/' + citation.id}>{citation.id}</a>
+          }
+          cite.pdf = {
+            _raw: true,
+            value: cite.pdf ? <a href={cite.pdf} rel='noopener noreferrer' target="_blank">[pdf]</a> : "no pdf"
+          }
+          return (
+            <li key={i}>
+              <TableObject
+                summary
+                object={cite}
+                tag={cite.title}
+              />
+            </li>
+          )
+        })}
+        </ul>
+      </div>
+    )
+  }
+}
+                // order={['id', 'pdf', 'year']}
+
+const mapStateToProps = state => ({
+  api: state.api
+})
+const mapDispatchToProps = dispatch => ({
+  actions: bindActionCreators({ ...actions }, dispatch),
+})
+
+export default connect(mapStateToProps, mapDispatchToProps)(PaperUnknown)
diff --git a/scraper/client/util.js b/scraper/client/util.js
index b00a0061..fe60d925 100644
--- a/scraper/client/util.js
+++ b/scraper/client/util.js
@@ -61,6 +61,8 @@ export const px = (n, w) => Math.round(n * w) + 'px'
 
 export const clamp = (n, a, b) => n < a ? a : n < b ? n : b
 
+export const choice = a => a[Math.floor(Math.random()*a.length)]
+
 /* URLs */
 
 export const hashPath = sha256 => {
diff --git a/scraper/s2-geocode-server.py b/scraper/s2-geocode-server.py
index c9219fe1..0a4dc06c 100644
--- a/scraper/s2-geocode-server.py
+++ b/scraper/s2-geocode-server.py
@@ -14,6 +14,8 @@ load_dotenv()
 from util import *
 
 locations_worksheet = fetch_worksheet('paper_locations')
+paper_lookup = fetch_google_lookup('citation_lookup')
+addresses = AddressBook()
 
 app = Flask(__name__, static_url_path="/reports", static_folder=os.path.abspath("reports"))
 
@@ -31,7 +33,6 @@ def page_not_found(e):
 
 @app.route('/api/institutions', methods=['GET'])
 def list_locations():
-  addresses = AddressBook()
   return jsonify({
     'entities': addresses.entities,
     'lookup': addresses.lookup,
@@ -39,10 +40,6 @@ def list_locations():
 
 @app.route('/api/papers', methods=['GET'])
 def list_papers():
-  lookup_keys, lines = fetch_google_sheet('citation_lookup')
-  paper_lookup = {}
-  for line in lines:
-    paper_lookup[line[0]] = line
   return jsonify({
     'papers': paper_lookup,
   })
@@ -50,7 +47,12 @@ def list_papers():
 @app.route('/api/address/<sha256>', methods=['GET'])
 def find_address(sha256):
   worksheet = fetch_worksheet('paper_locations')
-  cell = worksheet.find(sha256)
+  try:
+    cell = worksheet.find(sha256)
+  except:
+    return jsonify({
+      'error': 'no_match'
+    })
   if cell and cell.row:
     keys = worksheet.row_values(1)
     values_list = worksheet.row_values(cell.row)