From 83063b97105c7514d71ec2afaaf66def49116214 Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Wed, 13 Feb 2019 18:05:35 +0100 Subject: listing citations, known and unknown --- scraper/client/app.js | 6 +++- scraper/client/common/header.component.js | 2 +- scraper/client/common/table.component.js | 2 +- scraper/client/paper/index.js | 8 +++++ scraper/client/paper/paper.address.js | 38 ++++++++++++++++++++ scraper/client/paper/paper.citations.js | 58 +++++++++++++++++++++++++++++++ scraper/client/paper/paper.css | 16 +++++++++ scraper/client/paper/paper.random.js | 36 +++++++++++++++++++ scraper/client/paper/paper.unknown.js | 53 ++++++++++++++++++++++++++++ scraper/client/util.js | 2 ++ scraper/s2-geocode-server.py | 14 ++++---- 11 files changed, 226 insertions(+), 9 deletions(-) create mode 100644 scraper/client/paper/paper.address.js create mode 100644 scraper/client/paper/paper.citations.js create mode 100644 scraper/client/paper/paper.random.js create mode 100644 scraper/client/paper/paper.unknown.js (limited to 'scraper') diff --git a/scraper/client/app.js b/scraper/client/app.js index 4bc6276c..7600fe52 100644 --- a/scraper/client/app.js +++ b/scraper/client/app.js @@ -15,7 +15,12 @@ export default class App extends Component {
+ + + + +
@@ -25,7 +30,6 @@ export default class App extends Component { ) } } - // // // diff --git a/scraper/client/common/header.component.js b/scraper/client/common/header.component.js index a14d925f..0cd67c76 100644 --- a/scraper/client/common/header.component.js +++ b/scraper/client/common/header.component.js @@ -9,8 +9,8 @@ import * as actions from '../actions' class Header extends Component { componentDidMount() { - this.props.actions.getInstitutions() this.props.actions.getPapers() + this.props.actions.getInstitutions() } pickPaper(e) { diff --git a/scraper/client/common/table.component.js b/scraper/client/common/table.component.js index f9be0669..96b62835 100644 --- a/scraper/client/common/table.component.js +++ b/scraper/client/common/table.component.js @@ -15,7 +15,7 @@ export function TableObject({ tag, object, order, summary }) { let keys = Object.keys(object) if (order) { const grouped = keys.reduce((a, b) => { - if (summary && !object[b].trim().length) { + if (summary && !(object[b].trim ? object[b].trim().length : object[b])) { return a } const index = order.indexOf(b) diff --git a/scraper/client/paper/index.js b/scraper/client/paper/index.js index c10ea011..4fad0316 100644 --- a/scraper/client/paper/index.js +++ b/scraper/client/paper/index.js @@ -1,5 +1,9 @@ import Manager from './paper.manager' import Info from './paper.info' +import Citations from './paper.citations' +import UnknownCitations from './paper.unknown' +import Random from './paper.random' +import Address from './paper.address' import './paper.css' // import './search.css' @@ -7,4 +11,8 @@ import './paper.css' export { Manager, Info, + Citations, + UnknownCitations, + Random, + Address, } diff --git a/scraper/client/paper/paper.address.js b/scraper/client/paper/paper.address.js new file mode 100644 index 00000000..2240388f --- /dev/null +++ b/scraper/client/paper/paper.address.js @@ -0,0 +1,38 @@ +import React, { Component } from 'react' +import { bindActionCreators } from 'redux' +import { connect } from 'react-redux' + +import * as actions from '../actions' + +// import { Loader } from '../common' + +class PaperAddress extends Component { + componentDidMount() { + const { sha256 } = this.props.match.params + this.props.actions.getAddress(sha256) + } + + componentDidUpdate(newProps) { + const { sha256: oldSha256 } = this.props.match.params + const { sha256 } = newProps.match.params + if (sha256 !== oldSha256) { + this.props.actions.getPaperInfo(this.props.match.params.key) + } + } + + render() { + // if (!this.props.match.params.key) return null + // if (this.props.api.address.loading) return + // if (!this.props.api.paperInfo.dataset) return
Metadata not found
+ return null + } +} + +const mapStateToProps = state => ({ + api: state.api, +}) +const mapDispatchToProps = dispatch => ({ + actions: bindActionCreators({ ...actions }, dispatch), +}) + +export default connect(mapStateToProps, mapDispatchToProps)(PaperAddress) diff --git a/scraper/client/paper/paper.citations.js b/scraper/client/paper/paper.citations.js new file mode 100644 index 00000000..de423a77 --- /dev/null +++ b/scraper/client/paper/paper.citations.js @@ -0,0 +1,58 @@ +import React, { Component } from 'react' +import { bindActionCreators } from 'redux' +import { connect } from 'react-redux' + +import * as actions from '../actions' + +import { TableObject } from '../common' + +class PaperCitations extends Component { + render() { + const { dataset, citations } = this.props.api.paperInfo + if (!dataset || !citations) return null + console.log('rendering citations...') + return ( +
+

{dataset.name_full}: Citations

+
    + {citations.map((citation, i) => { + let cite = { ...citation } + cite.id = { + _raw: true, + value: {citation.id} + } + cite.pdf = { + _raw: true, + value: cite.pdf ? [pdf] : "no pdf" + } + cite.addresses = { + _raw: true, + value: cite.addresses.map((address, j) => ( +
    {address.address}{', '}{address.type}
    + )) + } + return ( +
  • + +
    • + ) + })} +
+
+ ) + } +} + +const mapStateToProps = state => ({ + api: state.api +}) +const mapDispatchToProps = dispatch => ({ + actions: bindActionCreators({ ...actions }, dispatch), +}) + +export default connect(mapStateToProps, mapDispatchToProps)(PaperCitations) diff --git a/scraper/client/paper/paper.css b/scraper/client/paper/paper.css index c1a775c1..cd2f8529 100644 --- a/scraper/client/paper/paper.css +++ b/scraper/client/paper/paper.css @@ -1,3 +1,19 @@ .paperInfo { padding: 10px; +} + +.citations ul { + list-style-type: none; + margin: 0; + padding: 0; +} + +.citations li { + list-style-type: none; + margin: 0; + padding-bottom: 40px; +} + +.type { + color: #888; } \ No newline at end of file diff --git a/scraper/client/paper/paper.random.js b/scraper/client/paper/paper.random.js new file mode 100644 index 00000000..aab22172 --- /dev/null +++ b/scraper/client/paper/paper.random.js @@ -0,0 +1,36 @@ +import React, { Component } from 'react' +// import { NavLink } from 'react-router-dom' +import { bindActionCreators } from 'redux' +import { connect } from 'react-redux' + +import { choice } from '../util' +import { history } from '../store' + +import * as actions from '../actions' + +class PaperRandom extends Component { + componentDidUpdate() { + const { citations } = this.props.api.unknownCitations + if (!citations) return + const citation = choice(citations) + console.log(citation) + if (citation.id) { + history.push('/paper/' + this.props.match.params.key + '/address/' + citation.id) + } + } + + render() { + return ( +
Sending you to a random citation...
+ ) + } +} + +const mapStateToProps = state => ({ + api: state.api +}) +const mapDispatchToProps = dispatch => ({ + actions: bindActionCreators({ ...actions }, dispatch), +}) + +export default connect(mapStateToProps, mapDispatchToProps)(PaperRandom) diff --git a/scraper/client/paper/paper.unknown.js b/scraper/client/paper/paper.unknown.js new file mode 100644 index 00000000..7a20f398 --- /dev/null +++ b/scraper/client/paper/paper.unknown.js @@ -0,0 +1,53 @@ +import React, { Component } from 'react' +import { bindActionCreators } from 'redux' +import { connect } from 'react-redux' + +import * as actions from '../actions' + +import { TableObject } from '../common' + +class PaperUnknown extends Component { + render() { + const { dataset } = this.props.api.paperInfo + const { citations } = this.props.api.unknownCitations + if (!dataset || !citations) return null + console.log('rendering unknown citations...') + return ( +
+

{dataset.name_full}: Unknown Citations

+
    + {citations.map((citation, i) => { + let cite = { ...citation } + cite.id = { + _raw: true, + value: {citation.id} + } + cite.pdf = { + _raw: true, + value: cite.pdf ? [pdf] : "no pdf" + } + return ( +
  • + +
    • + ) + })} +
+
+ ) + } +} + // order={['id', 'pdf', 'year']} + +const mapStateToProps = state => ({ + api: state.api +}) +const mapDispatchToProps = dispatch => ({ + actions: bindActionCreators({ ...actions }, dispatch), +}) + +export default connect(mapStateToProps, mapDispatchToProps)(PaperUnknown) diff --git a/scraper/client/util.js b/scraper/client/util.js index b00a0061..fe60d925 100644 --- a/scraper/client/util.js +++ b/scraper/client/util.js @@ -61,6 +61,8 @@ export const px = (n, w) => Math.round(n * w) + 'px' export const clamp = (n, a, b) => n < a ? a : n < b ? n : b +export const choice = a => a[Math.floor(Math.random()*a.length)] + /* URLs */ export const hashPath = sha256 => { diff --git a/scraper/s2-geocode-server.py b/scraper/s2-geocode-server.py index c9219fe1..0a4dc06c 100644 --- a/scraper/s2-geocode-server.py +++ b/scraper/s2-geocode-server.py @@ -14,6 +14,8 @@ load_dotenv() from util import * locations_worksheet = fetch_worksheet('paper_locations') +paper_lookup = fetch_google_lookup('citation_lookup') +addresses = AddressBook() app = Flask(__name__, static_url_path="/reports", static_folder=os.path.abspath("reports")) @@ -31,7 +33,6 @@ def page_not_found(e): @app.route('/api/institutions', methods=['GET']) def list_locations(): - addresses = AddressBook() return jsonify({ 'entities': addresses.entities, 'lookup': addresses.lookup, @@ -39,10 +40,6 @@ def list_locations(): @app.route('/api/papers', methods=['GET']) def list_papers(): - lookup_keys, lines = fetch_google_sheet('citation_lookup') - paper_lookup = {} - for line in lines: - paper_lookup[line[0]] = line return jsonify({ 'papers': paper_lookup, }) @@ -50,7 +47,12 @@ def list_papers(): @app.route('/api/address/', methods=['GET']) def find_address(sha256): worksheet = fetch_worksheet('paper_locations') - cell = worksheet.find(sha256) + try: + cell = worksheet.find(sha256) + except: + return jsonify({ + 'error': 'no_match' + }) if cell and cell.row: keys = worksheet.row_values(1) values_list = worksheet.row_values(cell.row) -- cgit v1.2.3-70-g09d2