diff options
25 files changed, 133 insertions, 84 deletions
@@ -170,3 +170,6 @@ site/public/user_content site/datasets/final/*.csv +flask.log +flask.log.* + diff --git a/client/index.js b/client/index.js index 668aebfb..5e36d341 100644 --- a/client/index.js +++ b/client/index.js @@ -65,14 +65,17 @@ function runApplets() { let opt = null payload.cmd = cmd payload.partz = cmdPartz + if (payload.cmd === 'load_file') { + payload.url = 'https://nyc3.digitaloceanspaces.com/megapixels/v1' + cmdPartz.shift() + return [el, payload] + } + if (payload.partz.length) { opt = payload.partz.shift().trim() if (opt.indexOf('http') === 0) { dataset = null url = opt } else if (opt.indexOf('assets') === 0) { - let pathname = window.location.pathname.replace('index.html', '') - url = 'https://nyc3.digitaloceanspaces.com/megapixels/v1' + pathname + opt dataset = null // console.log(url) } else { @@ -95,6 +98,7 @@ function runApplets() { } payload.dataset = dataset payload.url = url + console.log(payload) return [el, payload] }).filter(a => !!a) const withDataset = applets.map(a => a[1].dataset ? a[1] : null).filter(a => !!a) diff --git a/client/table/citations.table.js b/client/table/citations.table.js index f9599f5d..bbc55047 100644 --- a/client/table/citations.table.js +++ b/client/table/citations.table.js @@ -2,10 +2,10 @@ import React, { Component } from 'react' import { bindActionCreators } from 'redux' import { connect } from 'react-redux' import { ReactTabulator } from 'react-tabulator' -import MultiValueFormatter from "react-tabulator/lib/formatters/MultiValueFormatter" +import { saveAs } from 'file-saver' import { Loader } from '../common' -import { toArray, toTuples } from '../util' +import { toArray, toTuples, domainFromUrl } from '../util' export const citationsColumns = [ { title: 'Title', field: 'title', sorter: 'string' }, @@ -42,13 +42,7 @@ class CitationsTable extends Component { : (citation.doi && citation.doi.length) ? citation.doi[0] : 'https://www.semanticscholar.org/paper/' + citation.id - let pdf_text - const pdf_partz = pdf_link.split('/')[2].split('.') - if (pdf_partz.length > 2 && pdf_partz[pdf_partz.length - 2].length == 2) { - pdf_text = pdf_partz.slice(-3).join('.') - } else { - pdf_text = pdf_partz.slice(-2).join('.') - } + let pdf_text = domainFromUrl(pdf_link) return { title: citation.title, institution: citation.addresses.map(a => a.name).sort().join('; '), @@ -78,18 +72,47 @@ class CitationsTable extends Component { } } + download() { + const { formattedCitations } = this.state + const fn = this.props.payload.data.paper.key + '.csv' + const titles = citationsColumns.map(c => c.title) + const fields = citationsColumns.map(c => c.formatterParams ? c.formatterParams.urlField : c.field) + const rows = formattedCitations.map(citation => { + const row = fields.map(field => citation[field]).map(data => { + switch (typeof data) { + case 'number': + return String(data) + default: + return '\"' + String(data) + '\"' + } + }) + return row.join(",") + }) + + const blob = new Blob([ + [ + titles.join(','), + ...rows, + ].join('\n') + ], {type: "text/csv;charset=utf-8"}); + saveAs(blob, fn); + } + render() { const { formattedCitations, filteredCitations } = this.state if (!formattedCitations.length) return <Loader /> return ( <div className='citationBrowser'> - <input - type="text" - value={this.state.q} - onChange={e => this.updateFilter(e.target.value)} - className='q' - placeholder='Enter text to search citations...' - /> + <div className='citationHeader'> + <input + type="text" + value={this.state.q} + onChange={e => this.updateFilter(e.target.value)} + className='q' + placeholder='Enter text to search citations...' + /> + <span className='download' onClick={() => this.download()}>Download CSV</span> + </div> <ReactTabulator columns={citationsColumns} data={filteredCitations} diff --git a/client/table/file.table.js b/client/table/file.table.js index 92f5cf72..82c01ac5 100644 --- a/client/table/file.table.js +++ b/client/table/file.table.js @@ -1,25 +1,32 @@ import React, { Component } from 'react' import { bindActionCreators } from 'redux' import { connect } from 'react-redux' -import { toArray, toTuples } from '../util' +import { ReactTabulator } from 'react-tabulator' +import { toArray, toTuples, domainFromUrl } from '../util' import { Loader } from '../common' import csv from 'parse-csv' class FileTable extends Component { state = { - data: [] + keys: [], + data: [], + columns: [], } componentDidMount() { + const { payload } = this.props console.log(payload.url) fetch(payload.url, { mode: 'cors' }) .then(r => r.text()) .then(text => { try { + const keys = text.split('\n')[0].split(',').map(s => s.trim().replace(/\"/,'')) const data = csv.toJSON(text, { headers: { included: true } }) - this.setState({ data }) + // console.log(data) + const columns = this.getColumns(keys, data, payload.fields) + this.setState({ keys, data, columns }) } catch (e) { console.error("error making json:", payload.url) console.error(e) @@ -27,26 +34,41 @@ class FileTable extends Component { }) } - getColumns(payload) { - let { cmd, url, fields } = payload - return ((fields && fields.length) ? fields[0] : '').split(', ').map(field => { + getColumns(keys, data, fields) { + let titles = fields.length ? fields[0].split(', ') : keys + let numberFields = [] + let columns = keys.map((field, i) => { + const title = titles[i] || field + if (field.match('url')) { + let textField = field.replace('url', 'label') + data.forEach(el => el[textField] = domainFromUrl(el[field])) + return { + title, + field: textField, + formatter: 'link', + formatterParams: { target: "_blank", urlField: field, }, + sorter: 'string' + } + } switch (field) { + case 'images': + case 'year': + return { title, field: field.toLowerCase(), sorter: 'number' } default: - return { title: field, field: field.toLowerCase(), sorter: 'string' } + return { title, field: field.toLowerCase(), sorter: 'string' } } }) + return columns } render() { const { payload } = this.props - const { paper, citations } = payload.data - const columns = getColumns(payload) if (!this.state.data.length) { return <Loader /> } return ( <ReactTabulator - columns={citationsColumns} + columns={this.state.columns} data={this.state.data} options={{ height: 311, diff --git a/client/table/tabulator.css b/client/table/tabulator.css index c08ad580..17dad62a 100644 --- a/client/table/tabulator.css +++ b/client/table/tabulator.css @@ -30,9 +30,6 @@ } .citationBrowser { - padding: 10px; - border-radius: 4px; - background: #666; } .citationBrowser .q { max-width: 400px; @@ -42,4 +39,26 @@ background-repeat: no-repeat; box-shadow: 0px 2px 4px rgba(0,0,0,0.2); border: 0; +} + +.citationHeader { + width: 100%; + display: flex; + flex-direction: row; + align-items: flex-start; + justify-content: space-between; +} +span.download { + display: block; + font-size: 13px; + color: #ddd; + cursor: pointer; + background: #333; + padding: 5px 8px; + border-radius: 5px; + transition: all 0.2s; +} +.desktop span.download:hover { + color: #fff; + background: #666; }
\ No newline at end of file diff --git a/client/util/index.js b/client/util/index.js index 87d32ebb..e90e5466 100644 --- a/client/util/index.js +++ b/client/util/index.js @@ -63,6 +63,15 @@ export const percent = n => (n * 100).toFixed(1) + '%' export const px = (n, w) => Math.round(n * w) + 'px' export const clamp = (n, a, b) => n < a ? a : n < b ? n : b +export const domainFromUrl = url => { + const partz = url.split('/')[2].split('.') + if (partz.length > 2 && partz[partz.length - 2].length == 2) { + return partz.slice(-3).join('.') + } else { + return partz.slice(-2).join('.') + } +} + /* URLs */ export const preloadImage = opt => { diff --git a/package-lock.json b/package-lock.json index 4e9d6fac..43ee0cbf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3673,9 +3673,9 @@ } }, "file-saver": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/file-saver/-/file-saver-2.0.0.tgz", - "integrity": "sha512-cYM1ic5DAkg25pHKgi5f10ziAM7RJU37gaH1XQlyNDrtUnzhC/dfoV9zf2OmF0RMKi42jG5B0JWBnPQqyj/G6g==" + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/file-saver/-/file-saver-2.0.1.tgz", + "integrity": "sha512-dCB3K7/BvAcUmtmh1DzFdv0eXSVJ9IAFt1mw3XZfAexodNRoE29l3xB2EX4wH2q8m/UTzwzEPq/ArYk98kUkBQ==" }, "filename-regex": { "version": "2.0.1", @@ -8623,7 +8623,7 @@ }, "sprintf-js": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "resolved": "http://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", "dev": true }, diff --git a/package.json b/package.json index 6238e7e3..75fbebb7 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "exif-reader": "^1.0.2", "exifreader": "^2.5.0", "fetch-jsonp": "^1.1.3", - "file-saver": "^2.0.0-rc.3", + "file-saver": "^2.0.1", "history": "^4.7.2", "leaflet": "^1.3.4", "leaflet-arc": "^1.0.2", diff --git a/site/content/pages/test/assets/test.csv b/site/content/pages/test/assets/test.csv new file mode 100644 index 00000000..7156a814 --- /dev/null +++ b/site/content/pages/test/assets/test.csv @@ -0,0 +1,8 @@ +name,images,year,gender,description,url +aardvark,100,2019,m,bim da,https://asdf.us/ +bobcat,10,2017,f,in a tree,https://asdf.us/ +cow,20,2012,f,moooo,https://asdf.us/ +doe,2,2016,f,doe a deer,https://asdf.us/ +earwig,1,2017,m,just a bug,https://i.asdf.us/ +frog,17,2018,f,ribbit ribbit,https://i.asdf.us/ +giraffe,23,2009,m,i get around,https://adsf.us/ diff --git a/site/content/pages/test/csv.md b/site/content/pages/test/csv.md index b5f37754..85f714b4 100644 --- a/site/content/pages/test/csv.md +++ b/site/content/pages/test/csv.md @@ -15,6 +15,6 @@ authors: Megapixels ### [← Back to test index](/test/) ``` -load_file /datasets/lfw/assets/lfw_names_gender_kg_min.csv -Name, Images, Gender, Description +load_file /site/test/assets/test.csv +Name, Images, Year, Gender, Description, URL ``` diff --git a/site/includes/citations.html b/site/includes/citations.html index 32558d4a..5cd40a29 100644 --- a/site/includes/citations.html +++ b/site/includes/citations.html @@ -4,9 +4,6 @@ <p> The dataset citations used in the visualizations were collected from <a href="https://www.semanticscholar.org">Semantic Scholar</a>, a website which aggregates and indexes research papers. Each citation was geocoded using names of institutions found in the PDF front matter, or as listed on other resources. These papers have been manually verified to show that researchers downloaded and used the dataset to train or test machine learning algorithms. </p> - <p> - Add [button/link] to download CSV. Add search input field to filter. - </p> <div class="applet" data-payload="{"command": "citations"}"></div> </section>
\ No newline at end of file diff --git a/site/public/datasets/50_people_one_question/index.html b/site/public/datasets/50_people_one_question/index.html index 8e3d2d2b..540e2d0d 100644 --- a/site/public/datasets/50_people_one_question/index.html +++ b/site/public/datasets/50_people_one_question/index.html @@ -85,9 +85,6 @@ <p> The dataset citations used in the visualizations were collected from <a href="https://www.semanticscholar.org">Semantic Scholar</a>, a website which aggregates and indexes research papers. Each citation was geocoded using names of institutions found in the PDF front matter, or as listed on other resources. These papers have been manually verified to show that researchers downloaded and used the dataset to train or test machine learning algorithms. </p> - <p> - Add [button/link] to download CSV. Add search input field to filter. - </p> <div class="applet" data-payload="{"command": "citations"}"></div> </section> diff --git a/site/public/datasets/brainwash/index.html b/site/public/datasets/brainwash/index.html index c97349aa..5e8f3a4c 100644 --- a/site/public/datasets/brainwash/index.html +++ b/site/public/datasets/brainwash/index.html @@ -90,9 +90,6 @@ <p> The dataset citations used in the visualizations were collected from <a href="https://www.semanticscholar.org">Semantic Scholar</a>, a website which aggregates and indexes research papers. Each citation was geocoded using names of institutions found in the PDF front matter, or as listed on other resources. These papers have been manually verified to show that researchers downloaded and used the dataset to train or test machine learning algorithms. </p> - <p> - Add [button/link] to download CSV. Add search input field to filter. - </p> <div class="applet" data-payload="{"command": "citations"}"></div> </section><section> diff --git a/site/public/datasets/celeba/index.html b/site/public/datasets/celeba/index.html index e958cbef..f1ee0c22 100644 --- a/site/public/datasets/celeba/index.html +++ b/site/public/datasets/celeba/index.html @@ -85,9 +85,6 @@ <p> The dataset citations used in the visualizations were collected from <a href="https://www.semanticscholar.org">Semantic Scholar</a>, a website which aggregates and indexes research papers. Each citation was geocoded using names of institutions found in the PDF front matter, or as listed on other resources. These papers have been manually verified to show that researchers downloaded and used the dataset to train or test machine learning algorithms. </p> - <p> - Add [button/link] to download CSV. Add search input field to filter. - </p> <div class="applet" data-payload="{"command": "citations"}"></div> </section><section><h3>Research</h3> diff --git a/site/public/datasets/cofw/index.html b/site/public/datasets/cofw/index.html index 7ac30579..1f5aa315 100644 --- a/site/public/datasets/cofw/index.html +++ b/site/public/datasets/cofw/index.html @@ -95,9 +95,6 @@ To increase the number of training images, and since COFW has the exact same la <p> The dataset citations used in the visualizations were collected from <a href="https://www.semanticscholar.org">Semantic Scholar</a>, a website which aggregates and indexes research papers. Each citation was geocoded using names of institutions found in the PDF front matter, or as listed on other resources. These papers have been manually verified to show that researchers downloaded and used the dataset to train or test machine learning algorithms. </p> - <p> - Add [button/link] to download CSV. Add search input field to filter. - </p> <div class="applet" data-payload="{"command": "citations"}"></div> </section><section> diff --git a/site/public/datasets/duke_mtmc/index.html b/site/public/datasets/duke_mtmc/index.html index 9664181e..83050506 100644 --- a/site/public/datasets/duke_mtmc/index.html +++ b/site/public/datasets/duke_mtmc/index.html @@ -103,9 +103,6 @@ <p> The dataset citations used in the visualizations were collected from <a href="https://www.semanticscholar.org">Semantic Scholar</a>, a website which aggregates and indexes research papers. Each citation was geocoded using names of institutions found in the PDF front matter, or as listed on other resources. These papers have been manually verified to show that researchers downloaded and used the dataset to train or test machine learning algorithms. </p> - <p> - Add [button/link] to download CSV. Add search input field to filter. - </p> <div class="applet" data-payload="{"command": "citations"}"></div> </section><section><h2>Research Notes</h2> diff --git a/site/public/datasets/hrt_transgender/index.html b/site/public/datasets/hrt_transgender/index.html index ed36abb5..528d1c3d 100644 --- a/site/public/datasets/hrt_transgender/index.html +++ b/site/public/datasets/hrt_transgender/index.html @@ -97,9 +97,6 @@ <p> The dataset citations used in the visualizations were collected from <a href="https://www.semanticscholar.org">Semantic Scholar</a>, a website which aggregates and indexes research papers. Each citation was geocoded using names of institutions found in the PDF front matter, or as listed on other resources. These papers have been manually verified to show that researchers downloaded and used the dataset to train or test machine learning algorithms. </p> - <p> - Add [button/link] to download CSV. Add search input field to filter. - </p> <div class="applet" data-payload="{"command": "citations"}"></div> </section> diff --git a/site/public/datasets/index.html b/site/public/datasets/index.html index 1d2630e1..3a2dbd52 100644 --- a/site/public/datasets/index.html +++ b/site/public/datasets/index.html @@ -42,8 +42,8 @@ <span class='title'>Brainwash</span> <div class='fields'> <div class='year visible'><span>2015</span></div> - <div class='purpose'><span>Decoding image into set of people detections.</span></div> - <div class='images'><span>11,918 images</span></div> + <div class='purpose'><span>Head detection</span></div> + <div class='images'><span>11,917 images</span></div> <div class='identities'><span></span></div> </div> </div> @@ -90,7 +90,7 @@ <span class='title'>MS Celeb</span> <div class='fields'> <div class='year visible'><span>2016</span></div> - <div class='purpose'><span>face recognition</span></div> + <div class='purpose'><span>Large-scale face recognition</span></div> <div class='images'><span>1,000,000 images</span></div> <div class='identities'><span>100,000 </span></div> </div> @@ -114,9 +114,9 @@ <span class='title'>Unconstrained College Students</span> <div class='fields'> <div class='year visible'><span>2018</span></div> - <div class='purpose'><span>Unconstrained face recognition</span></div> + <div class='purpose'><span>Face recognition, face detection</span></div> <div class='images'><span>16,149 images</span></div> - <div class='identities'><span>4,362 </span></div> + <div class='identities'><span>1,732 </span></div> </div> </div> </a> @@ -126,7 +126,7 @@ <span class='title'>VIPeR</span> <div class='fields'> <div class='year visible'><span>2007</span></div> - <div class='purpose'><span>pedestrian re-identification</span></div> + <div class='purpose'><span>Person re-identification</span></div> <div class='images'><span>1,264 images</span></div> <div class='identities'><span>632 </span></div> </div> diff --git a/site/public/datasets/lfw/index.html b/site/public/datasets/lfw/index.html index 22384d77..5f076fc7 100644 --- a/site/public/datasets/lfw/index.html +++ b/site/public/datasets/lfw/index.html @@ -113,9 +113,6 @@ <p> The dataset citations used in the visualizations were collected from <a href="https://www.semanticscholar.org">Semantic Scholar</a>, a website which aggregates and indexes research papers. Each citation was geocoded using names of institutions found in the PDF front matter, or as listed on other resources. These papers have been manually verified to show that researchers downloaded and used the dataset to train or test machine learning algorithms. </p> - <p> - Add [button/link] to download CSV. Add search input field to filter. - </p> <div class="applet" data-payload="{"command": "citations"}"></div> </section><section><h3>Commercial Use</h3> diff --git a/site/public/datasets/market_1501/index.html b/site/public/datasets/market_1501/index.html index 9a05d20e..951646e3 100644 --- a/site/public/datasets/market_1501/index.html +++ b/site/public/datasets/market_1501/index.html @@ -83,9 +83,6 @@ <p> The dataset citations used in the visualizations were collected from <a href="https://www.semanticscholar.org">Semantic Scholar</a>, a website which aggregates and indexes research papers. Each citation was geocoded using names of institutions found in the PDF front matter, or as listed on other resources. These papers have been manually verified to show that researchers downloaded and used the dataset to train or test machine learning algorithms. </p> - <p> - Add [button/link] to download CSV. Add search input field to filter. - </p> <div class="applet" data-payload="{"command": "citations"}"></div> </section><section><h2>Research Notes</h2> diff --git a/site/public/datasets/msceleb/index.html b/site/public/datasets/msceleb/index.html index 0ddf0c68..9a671c8e 100644 --- a/site/public/datasets/msceleb/index.html +++ b/site/public/datasets/msceleb/index.html @@ -101,9 +101,6 @@ <p> The dataset citations used in the visualizations were collected from <a href="https://www.semanticscholar.org">Semantic Scholar</a>, a website which aggregates and indexes research papers. Each citation was geocoded using names of institutions found in the PDF front matter, or as listed on other resources. These papers have been manually verified to show that researchers downloaded and used the dataset to train or test machine learning algorithms. </p> - <p> - Add [button/link] to download CSV. Add search input field to filter. - </p> <div class="applet" data-payload="{"command": "citations"}"></div> </section><section><h3>Additional Information</h3> diff --git a/site/public/datasets/pipa/index.html b/site/public/datasets/pipa/index.html index 9e7eb164..fe6a4742 100644 --- a/site/public/datasets/pipa/index.html +++ b/site/public/datasets/pipa/index.html @@ -83,9 +83,6 @@ <p> The dataset citations used in the visualizations were collected from <a href="https://www.semanticscholar.org">Semantic Scholar</a>, a website which aggregates and indexes research papers. Each citation was geocoded using names of institutions found in the PDF front matter, or as listed on other resources. These papers have been manually verified to show that researchers downloaded and used the dataset to train or test machine learning algorithms. </p> - <p> - Add [button/link] to download CSV. Add search input field to filter. - </p> <div class="applet" data-payload="{"command": "citations"}"></div> </section><section><h2>Research Notes</h2> diff --git a/site/public/datasets/uccs/index.html b/site/public/datasets/uccs/index.html index 2477c9f8..10b7603e 100644 --- a/site/public/datasets/uccs/index.html +++ b/site/public/datasets/uccs/index.html @@ -98,9 +98,6 @@ <p> The dataset citations used in the visualizations were collected from <a href="https://www.semanticscholar.org">Semantic Scholar</a>, a website which aggregates and indexes research papers. Each citation was geocoded using names of institutions found in the PDF front matter, or as listed on other resources. These papers have been manually verified to show that researchers downloaded and used the dataset to train or test machine learning algorithms. </p> - <p> - Add [button/link] to download CSV. Add search input field to filter. - </p> <div class="applet" data-payload="{"command": "citations"}"></div> </section><section><h3>Research Notes</h3> diff --git a/site/public/datasets/viper/index.html b/site/public/datasets/viper/index.html index e94568a3..cc4272c8 100644 --- a/site/public/datasets/viper/index.html +++ b/site/public/datasets/viper/index.html @@ -100,9 +100,6 @@ <p> The dataset citations used in the visualizations were collected from <a href="https://www.semanticscholar.org">Semantic Scholar</a>, a website which aggregates and indexes research papers. Each citation was geocoded using names of institutions found in the PDF front matter, or as listed on other resources. These papers have been manually verified to show that researchers downloaded and used the dataset to train or test machine learning algorithms. </p> - <p> - Add [button/link] to download CSV. Add search input field to filter. - </p> <div class="applet" data-payload="{"command": "citations"}"></div> </section> diff --git a/site/public/test/csv/index.html b/site/public/test/csv/index.html index 3257716f..70a7d257 100644 --- a/site/public/test/csv/index.html +++ b/site/public/test/csv/index.html @@ -28,7 +28,7 @@ <section><h1>CSV Test</h1> <h3><a href="/test/">← Back to test index</a></h3> -</section><section class='applet_container'><div class='applet' data-payload='{"command": "load_file /datasets/lfw/assets/lfw_names_gender_kg_min.csv", "fields": ["Name, Images, Gender, Description"]}'></div></section> +</section><section class='applet_container'><div class='applet' data-payload='{"command": "load_file /site/test/assets/test.csv", "fields": ["Name, Images, Year, Gender, Description, URL"]}'></div></section> </div> <footer> |