summaryrefslogtreecommitdiff
path: root/scraper
diff options
context:
space:
mode:
Diffstat (limited to 'scraper')
-rw-r--r--scraper/client/actions.js4
-rw-r--r--scraper/client/app.js2
-rw-r--r--scraper/client/paper/paper.citations.js41
-rw-r--r--scraper/client/paper/paper.css64
-rw-r--r--scraper/client/paper/paper.random.js2
-rw-r--r--scraper/client/paper/paper.verify.js180
-rw-r--r--scraper/client/store.js2
-rw-r--r--scraper/client/types.js6
-rw-r--r--scraper/s2-geocode-server.py28
9 files changed, 267 insertions, 62 deletions
diff --git a/scraper/client/actions.js b/scraper/client/actions.js
index 6c4e16b2..734a8cbd 100644
--- a/scraper/client/actions.js
+++ b/scraper/client/actions.js
@@ -32,6 +32,10 @@ export const postAddress = data => dispatch => {
api(dispatch, post, 'address', '/api/address/add', data)
}
+export const getVerifications = () => dispatch => (
+ api(dispatch, get, 'verifications', '/api/verifications', {})
+)
+
export const getVerification = sha256 => dispatch => (
api(dispatch, get, 'verify', '/api/verify/' + sha256, {})
)
diff --git a/scraper/client/app.js b/scraper/client/app.js
index dc5c6fe6..b449d0d0 100644
--- a/scraper/client/app.js
+++ b/scraper/client/app.js
@@ -15,7 +15,7 @@ export default class App extends Component {
<div className='body'>
<Route path="/paper/:key/" component={Paper.Manager} />
<Switch>
- <Route exact path="/paper/:key/" component={Paper.Random} />
+ <Route exact path="/paper/:key/" component={Paper.Info} />
<Route exact path="/paper/:key/citations/" component={Paper.Citations} />
<Route exact path="/paper/:key/unknown/" component={Paper.UnknownCitations} />
<Route exact path="/paper/:key/info/" component={Paper.Info} />
diff --git a/scraper/client/paper/paper.citations.js b/scraper/client/paper/paper.citations.js
index 41ddd55e..5c8e0e0c 100644
--- a/scraper/client/paper/paper.citations.js
+++ b/scraper/client/paper/paper.citations.js
@@ -6,13 +6,20 @@ import { Link } from 'react-router-dom'
import * as actions from '../actions'
import { TableObject } from '../common'
+import { USES_DATASET } from '../types'
class PaperCitations extends Component {
+ componentDidMount() {
+ this.props.actions.getVerifications()
+ }
+
render() {
const { dataset, citations } = this.props.api.paperInfo
+ let { verifications } = this.props.api.verifications
+ verifications = verifications || {}
if (!dataset || !citations) return null
console.log('rendering citations...')
- console.log(citations)
+ // console.log(citations)
return (
<div className='citations'>
<h2>{dataset.name_full}: Citations</h2>
@@ -27,19 +34,49 @@ class PaperCitations extends Component {
_raw: true,
value: (cite.pdf && cite.pdf.length) ? <a href={cite.pdf[0]} rel='noopener noreferrer' target="_blank">[pdf]</a> : "no pdf"
}
+ cite.s2 = {
+ _raw: true,
+ value: <a
+ href={'https://www.semanticscholar.org/paper/' + citation.id}
+ target="_blank"
+ rel="noopener noreferrer"
+ className={'pdfLink'}
+ >{'[semantic scholar]'}</a>
+ }
cite.addresses = {
_raw: true,
value: cite.addresses.map((address, j) => (
<div key={j}>{address.name}{', '}<span className='type'>{address.type}</span></div>
))
}
+ if (citation.id in verifications && verifications[citation.id].dataset === dataset.key) {
+ const verification = verifications[citation.id]
+ cite.verified = {
+ _raw: true,
+ value: verification.uses_dataset === USES_DATASET.YES
+ ? <span className='verified'>uses dataset</span>
+ : verification.uses_dataset === USES_DATASET.NO
+ ? <span className='unverified'>{"doesn't use dataset"}</span>
+ : verification.uses_dataset === 'TRUE'
+ ? <span className='verified'>uses dataset</span>
+ : verification.uses_dataset === 'FALSE'
+ ? <span className='unverified'>{"doesn't use dataset"}</span>
+ : <span className='unknown'>{"not enough information"}</span>
+ }
+ }
+ else {
+ cite.verified = {
+ _raw: true,
+ value: <span className='unknown'>unknown</span>
+ }
+ }
return (
<li key={i}>
<TableObject
summary
object={cite}
tag={cite.title}
- order={['id', 'pdf', 'year', 'addresses']}
+ order={['id', 'pdf', 's2', 'year', 'addresses', 'verified']}
/>
</li>
)
diff --git a/scraper/client/paper/paper.css b/scraper/client/paper/paper.css
index 4815bd30..025d6b8a 100644
--- a/scraper/client/paper/paper.css
+++ b/scraper/client/paper/paper.css
@@ -1,8 +1,20 @@
+.row {
+ display: flex;
+ flex-direction: row;
+ padding: 4px;
+}
+.rowHeading {
+ display: block;
+ width: 194px;
+}
.form, .paperInfo {
padding: 10px;
width: 100%;
}
+.citations {
+ padding:40px;
+}
.citations ul {
list-style-type: none;
margin: 0;
@@ -12,7 +24,13 @@
.citations li {
list-style-type: none;
margin: 0;
- padding-bottom: 40px;
+ padding-bottom: 20px;
+ border:1px solid #ccc;
+}
+.citations h3{
+ font-size:20px;
+ margin:10px 0 15px 0;
+ font-weight: bold;
}
.type {
@@ -26,6 +44,32 @@
cursor: pointer;
color: #11f;
}
+.verified {
+ font-weight: bold;
+ color: white;
+ font-weight: bold;
+ display: inline-block;
+ background: #00b200;
+ padding:4px;
+ font-size:12px;
+}
+.unverified {
+ color: white;
+ font-weight: bold;
+ display: inline-block;
+ background: #ff0000;
+ padding:4px;
+ font-size:12px;
+}
+.unknown {
+ font-weight: bold;
+ color: white;
+ font-weight: bold;
+ display: inline-block;
+ background: #999;
+ padding:4px;
+ font-size:12px;
+}
.param {
display: flex;
@@ -39,9 +83,16 @@ input.notes {
.param .btn {
margin-top: 5px;
}
-.vetting {
- width: 250px;
- margin-right: 10px;
+.row.disabled { opacity: 0.5; pointer-events: none;}
+
+.vettingRow label {
+ display: flex;
+ justify-content: center;
+ align-items: center;
+ padding: 0 10px 0 0;
+}
+.vettingRow input {
+ margin-right: 6px;
}
.param label {
display: block;
@@ -51,7 +102,11 @@ input.notes {
.param input[type=checkbox] {
margin: 6px 0;
}
+input[type=text] {
+ margin-bottom: 2px;
+}
textarea {
+ margin-bottom: 2px;
padding: 4px;
font-size: 14px;
}
@@ -68,6 +123,7 @@ iframe.pdfViewer {
header select {
margin-right: 10px;
width: 100px;
+ min-width: 200px;
}
header a {
color: white;
diff --git a/scraper/client/paper/paper.random.js b/scraper/client/paper/paper.random.js
index aab22172..c7476332 100644
--- a/scraper/client/paper/paper.random.js
+++ b/scraper/client/paper/paper.random.js
@@ -15,7 +15,7 @@ class PaperRandom extends Component {
const citation = choice(citations)
console.log(citation)
if (citation.id) {
- history.push('/paper/' + this.props.match.params.key + '/address/' + citation.id)
+ history.push('/paper/' + this.props.match.params.key + '/verify/' + citation.id)
}
}
diff --git a/scraper/client/paper/paper.verify.js b/scraper/client/paper/paper.verify.js
index 5f85a551..65342adf 100644
--- a/scraper/client/paper/paper.verify.js
+++ b/scraper/client/paper/paper.verify.js
@@ -3,16 +3,15 @@ import { bindActionCreators } from 'redux'
import { connect } from 'react-redux'
import * as actions from '../actions'
-
import { history } from '../store'
-
-import { Loader, Autocomplete } from '../common'
+import { Loader } from '../common'
+import { USES_DATASET } from '../types'
const initialState = {
citation: null,
- uses_dataset: false,
- doesnt_use_dataset: true,
- images_in_paper: false,
+ uses_dataset: USES_DATASET.UNKNOWN,
+ images_in_paper: "FALSE",
+ used_as_model: "FALSE",
verified_by: localStorage.getItem('verify.username') || '',
reference: '',
notes: '',
@@ -64,7 +63,6 @@ class PaperVerify extends Component {
this.setState({
...citationState,
uses_dataset: paper.uses_dataset === "TRUE",
- doesnt_use_dataset: paper.doesnt_use_dataset === "TRUE",
images_in_paper: paper.images_in_paper === "TRUE",
verified_by: paper.verified_by,
reference: paper.reference,
@@ -103,8 +101,8 @@ class PaperVerify extends Component {
title: this.state.citation.title,
dataset: this.props.api.paperInfo.dataset.key,
uses_dataset: this.state.uses_dataset,
- doesnt_use_dataset: this.state.doesnt_use_dataset,
images_in_paper: this.state.images_in_paper,
+ used_as_model: this.state.used_as_model,
verified_by: this.state.verified_by,
reference: this.state.reference,
notes: this.state.notes,
@@ -169,6 +167,7 @@ class PaperVerify extends Component {
return (
<a
key={i}
+ href={pdf}
onClick={() => this.setState({ pdf_index: i })}
className={i === this.state.pdf_index ? 'selected pdfLink' : 'pdfLink'}
>
@@ -186,45 +185,115 @@ class PaperVerify extends Component {
</a>
</div>
- <div className='param'>
- <label>Uses the dataset?</label>
- <input
- className='vetting'
- type='radio'
- name='uses_dataset'
- checked={!!this.state.uses_dataset}
- onChange={e => this.setState({
- uses_dataset: e.target.checked,
- doesnt_use_dataset: !e.target.checked,
- })}
- />
+ <div className='row vettingRow'>
+ <div className='rowHeading'>
+ {'Uses dataset'}
+ </div>
+ <label>
+ <input
+ className='vetting'
+ type='radio'
+ name='uses_dataset'
+ checked={this.state.uses_dataset === USES_DATASET.YES || this.state.uses_dataset === "FALSE"}
+ onChange={e => this.setState({
+ uses_dataset: USES_DATASET.YES,
+ })}
+ />
+ <div>{"Yes"}</div>
+ </label>
+
+ <label>
+ <input
+ className='vetting'
+ type='radio'
+ name='uses_dataset'
+ checked={this.state.uses_dataset === USES_DATASET.NO || this.state.uses_dataset === "FALSE"}
+ onChange={e => this.setState({
+ uses_dataset: USES_DATASET.NO,
+ })}
+ />
+ <div>{"No"}</div>
+ </label>
+
+ <label>
+ <input
+ className='vetting'
+ type='radio'
+ name='uses_dataset'
+ checked={this.state.uses_dataset === USES_DATASET.UNKNOWN || this.state.uses_dataset === "FALSE"}
+ onChange={e => this.setState({
+ uses_dataset: USES_DATASET.UNKNOWN,
+ })}
+ />
+ <div>{"Not enough information"}</div>
+ </label>
</div>
- <div className='param'>
- <label>{"Doesn't use dataset"}</label>
- <input
- className='vetting'
- type='radio'
- name='uses_dataset'
- checked={!!this.state.doesnt_use_dataset}
- onChange={e => this.setState({
- uses_dataset: !e.target.checked,
- doesnt_use_dataset: e.target.checked
- })}
- />
+ <div
+ className={this.state.uses_dataset === USES_DATASET.YES ? 'row vettingRow' : 'row vettingRow disabled'}
+ disabled={this.state.uses_dataset === USES_DATASET.YES ? false : true}
+ >
+ <div className='rowHeading'>
+ {'Paper shows images'}
+ </div>
+
+ <label>
+ <input
+ className='vetting'
+ type='radio'
+ name='images_in_paper'
+ checked={this.state.images_in_paper === "TRUE"}
+ onChange={e => this.setState({
+ images_in_paper: "TRUE",
+ })}
+ />
+ <div>{"Yes"}</div>
+ </label>
+
+ <label>
+ <input
+ className='vetting'
+ type='radio'
+ name='images_in_paper'
+ checked={this.state.images_in_paper === "FALSE"}
+ onChange={e => this.setState({
+ images_in_paper: "FALSE",
+ })}
+ />
+ <div>{"No"}</div>
+ </label>
</div>
- <div className='param'>
- <label>{"Uses images from dataset"}</label>
- <input
- className='vetting'
- type='checkbox'
- name='images_in_paper'
- checked={!!this.state.images_in_paper}
- onChange={e => this.setState({
- images_in_paper: e.target.checked,
- })}
- />
+ <div className='row vettingRow'>
+ <div className='rowHeading'>
+ {'Used as model'}
+ </div>
+
+ <label>
+ <input
+ className='vetting'
+ type='radio'
+ name='used_as_model'
+ checked={this.state.used_as_model === "TRUE"}
+ onChange={e => this.setState({
+ used_as_model: "TRUE",
+ })}
+ />
+ <div>{"Yes"}</div>
+ </label>
+
+ <label>
+ <input
+ className='vetting'
+ type='radio'
+ name='used_as_model'
+ checked={this.state.used_as_model === "FALSE"}
+ onChange={e => this.setState({
+ used_as_model: "FALSE",
+ })}
+ />
+ <div>{"No"}</div>
+ </label>
</div>
<div className='param'>
@@ -244,10 +313,10 @@ class PaperVerify extends Component {
rows={3}
cols={50}
type='text'
- className='notes'
- value={this.state.notes}
- placeholder='Notes'
- onChange={e => this.setState({ notes: e.target.value })}
+ className='reference'
+ value={this.state.reference}
+ placeholder='Reference'
+ onChange={e => this.setState({ reference: e.target.value })}
/>
</div>
@@ -274,7 +343,7 @@ class PaperVerify extends Component {
<button
className='btn'
onClick={this.prev.bind(this)}
- >{'Prev >'}</button>
+ >{'< Prev'}</button>
<button
className='btn'
@@ -288,6 +357,21 @@ class PaperVerify extends Component {
}
}
+/*
+ <div className='param'>
+ <label>{"Verifiable"}</label>
+ <input
+ className='vetting'
+ type='checkbox'
+ name='verifiable'
+ checked={!!this.state.verifiable}
+ onChange={e => this.setState({
+ verifiable: e.target.checked,
+ })}
+ />
+ </div>
+*/
+
const mapStateToProps = state => ({
api: state.api,
})
diff --git a/scraper/client/store.js b/scraper/client/store.js
index 8649d472..b199ae29 100644
--- a/scraper/client/store.js
+++ b/scraper/client/store.js
@@ -13,6 +13,8 @@ const initialState = () => ({
address: {},
paperInfo: {},
unknownCitations: {},
+ verify: {},
+ verifications: {},
options: {}
})
diff --git a/scraper/client/types.js b/scraper/client/types.js
index 53c0cff0..95b7a4e3 100644
--- a/scraper/client/types.js
+++ b/scraper/client/types.js
@@ -11,3 +11,9 @@ export const api = tagAsType('api', [
])
export const init = '@@INIT'
+
+export const USES_DATASET = {
+ YES: "1",
+ NO: "-1",
+ UNKNOWN: "0",
+}
diff --git a/scraper/s2-geocode-server.py b/scraper/s2-geocode-server.py
index 9d824443..67da62d7 100644
--- a/scraper/s2-geocode-server.py
+++ b/scraper/s2-geocode-server.py
@@ -13,9 +13,6 @@ load_dotenv()
from util import *
-locations_worksheet = fetch_worksheet('paper_locations')
-verifications_worksheet = fetch_worksheet('verifications')
-paper_lookup = fetch_google_lookup('citation_lookup')
addresses = AddressBook()
app = Flask(__name__, static_url_path="/reports", static_folder=os.path.abspath("reports"))
@@ -41,6 +38,7 @@ def list_locations():
@app.route('/api/papers', methods=['GET'])
def list_papers():
+ paper_lookup = fetch_google_lookup('citation_lookup')
return jsonify({
'papers': paper_lookup,
})
@@ -73,6 +71,7 @@ def add_address():
form = request.get_json()
print(form)
# id, title, institution_1, institution_2, institution_3, institution_4, notes
+ locations_worksheet = fetch_worksheet('paper_locations')
locations_worksheet.append_row([
form['paper_id'],
form['title'],
@@ -90,6 +89,12 @@ def add_address():
'status': 'ok'
})
+@app.route('/api/verifications', methods=['GET'])
+def list_verifications():
+ return jsonify({
+ 'verifications': fetch_google_lookup('verifications', item_key='paper_id'),
+ })
+
@app.route('/api/verify/<sha256>', methods=['GET'])
def find_verification(sha256):
worksheet = fetch_worksheet('verifications')
@@ -114,12 +119,23 @@ def find_verification(sha256):
})
@app.route('/api/verify/add', methods=['POST'])
-def add_verifications():
+def add_verification():
+ worksheet = fetch_worksheet('verifications')
form = request.get_json()
print(form)
- keys = verifications_worksheet.row_values(1)
+ keys = worksheet.row_values(1)
row = [ form[key] if key in form else '' for key in keys ]
- verifications_worksheet.append_row(row)
+ try:
+ cell = worksheet.find(form['paper_id'])
+ except:
+ cell = None
+
+ if cell:
+ for item, i in enumerate(row):
+ worksheet.update_cell(cell.row, i+1, item)
+ else:
+ worksheet.append_row(row)
+
return jsonify({
'status': 'ok'
})
generated by cgit v1.2.3-70-g09d2 (git 2.51.0) at 2025-12-11 22:59:24 +0000