diff options
27 files changed, 489 insertions, 115 deletions
@@ -9,12 +9,29 @@ FaceQuery.me, mozilla, nytimes - nvm, node ``` +conda install pytorch torchvision -c pytorch +conda install faiss-cpu -c pytorch +pip install numpy Pillow +pip install dlib +pip install requests simplejson click pdfminer.six +pip install urllib3 flask flask_sqlalchemy mysql-connector +pip install pymediainfo tqdm opencv-python imutils +pip install scikit-image python-dotenv imagehash scikit-learn colorlog + sudo apt-get install libmysqlclient-dev mkdir -p /data_store_hdd/apps/megapixels/faiss/indexes mkdir -p /data_store_hdd/apps/megapixels/faiss/metadata ``` +### MySQL note + +You may need to set the database charset to `utf8mb4` in order to import the CSVs: + +``` +ALTER DATABASE megapixels CHARACTER SET = utf8mb4 COLLATE = utf8mb4_unicode_ci; +``` + ## Building the site ``` diff --git a/client/actions.js b/client/actions.js index bb011838..2be8229d 100644 --- a/client/actions.js +++ b/client/actions.js @@ -1,5 +1,7 @@ import * as faceSearch from './faceSearch/faceSearch.actions' +import * as nameSearch from './nameSearch/nameSearch.actions' export { - faceSearch + faceSearch, + nameSearch, } diff --git a/client/applet.js b/client/applet.js index 4d2a8e6c..80d40657 100644 --- a/client/applet.js +++ b/client/applet.js @@ -1,13 +1,16 @@ import React, { Component } from 'react' import { Container as FaceSearchContainer } from './faceSearch' +import { Container as NameSearchContainer } from './nameSearch' export default class Applet extends Component { render() { - console.log(this.props) + // console.log(this.props) switch (this.props.payload.cmd) { case 'face_search': return <FaceSearchContainer {...this.props} /> + case 'name_search': + return <NameSearchContainer {...this.props} /> default: return <pre style={{ color: '#0f0' }}>{'Megapixels'}</pre> } diff --git a/client/faceSearch/faceSearch.actions.js b/client/faceSearch/faceSearch.actions.js index 224977b5..03e1a91d 100644 --- a/client/faceSearch/faceSearch.actions.js +++ b/client/faceSearch/faceSearch.actions.js @@ -8,7 +8,7 @@ import { post, preloadImage } from '../util' // urls const url = { - upload: (dataset) => process.env.API_HOST + '/api/dataset/' + dataset + '/face/', + upload: (dataset) => process.env.API_HOST + '/api/dataset/' + dataset + '/face', } export const publicUrl = { } diff --git a/client/faceSearch/faceSearch.container.js b/client/faceSearch/faceSearch.container.js index f96961db..94c6eb9f 100644 --- a/client/faceSearch/faceSearch.container.js +++ b/client/faceSearch/faceSearch.container.js @@ -10,7 +10,7 @@ import FaceSearchResult from './faceSearch.result' class FaceSearchContainer extends Component { render() { const { payload } = this.props - console.log(payload) + // console.log(payload) return ( <div className='searchContainer'> <FaceSearchQuery payload={payload} /> diff --git a/client/faceSearch/faceSearch.result.js b/client/faceSearch/faceSearch.result.js index 1882def0..936bc8d2 100644 --- a/client/faceSearch/faceSearch.result.js +++ b/client/faceSearch/faceSearch.result.js @@ -4,51 +4,97 @@ import { connect } from 'react-redux' import { courtesyS } from '../util' import * as actions from './faceSearch.actions' +import { Loader } from '../common' const errors = { - 'bbox': "Sorry, we couldn't find a face in that image. Please choose an image where the face is large and clear.", - 'nomatch': "We didn't find a match.", - 'default': "There was an error!", + bbox: ( + <div> + <h2>No face found</h2> + {"Sorry, we didn't detect a face in that image. "} + {"Please choose an image where the face is large and clear."} + </div> + ), + nomatch: ( + <div> + <h2>{"You're clear"}</h2> + {"No images in this dataset match your face. We show only matches above 70% probability."} + </div> + ), + error: ( + <div> + <h2>{"No matches found"}</h2> + {"Sorry, an error occured."} + </div> + ), } class FaceSearchResult extends Component { render() { const { dataset } = this.props.payload - const { distances, results, error } = this.props.result + const { query, distances, results, loading, error } = this.props.result + console.log(this.props.result) + if (loading) { + return ( + <div className='result'> + <div> + <Loader /><br /> + <h2>Searching...</h2> + </div> + </div> + ) + } if (error) { - let errorMessage = errors[error.message] || errors.default + // console.log(error) + let errorMessage = errors[error] || errors.error return ( <div className='result'>{errorMessage}</div> ) } if (!results) { - return ( - <div className='result'></div> - ) + return <div className='result'></div> } - if (!this.props.result.results.length) { + if (!results.length) { return ( - <div className='result'>No results</div> + <div className='result'>{errors.nomatch}</div> ) } const els = results.map((result, i) => { const distance = distances[i] const { uuid } = result.uuid + const { x, y, w, h } = result.roi const { fullname, gender, description, images } = result.identity + const bbox = { + left: (100 * x) + '%', + top: (100 * y) + '%', + width: (100 * w) + '%', + height: (100 * h) + '%', + } + // console.log(bbox) return ( - <div> - <img src={'https://megapixels.nyc3.digitaloceanspaces.com/v1/media/' + dataset + '/' + uuid + '.jpg'} /> + <div key={i}> + <div className='img'> + <img src={'https://megapixels.nyc3.digitaloceanspaces.com/v1/media/' + dataset + '/' + uuid + '.jpg'} /> + <div className='bbox' style={bbox} /> + </div> {fullname} {'('}{gender}{')'}<br/> {description}<br/> - {courtesyS(images, 'image')}<br /> - {distance} + {courtesyS(images, 'image')}{' in dataset'}<br /> + {Math.round((1 - distance) * 100)}{'% match'} </div> ) }) return ( <div className='result'> - <div class='results'> + <div className="about"> + <h2>Did we find you?</h2> + {'These faces matched images in the '} + <b><tt>{dataset}</tt></b> + {' dataset with over 70% probability.'} + <br /> + <small>Query took {query.timing.toFixed(2)} seconds</small> + </div> + <div className='results'> {els} </div> </div> diff --git a/client/index.js b/client/index.js index 2beb5526..93341a77 100644 --- a/client/index.js +++ b/client/index.js @@ -38,7 +38,6 @@ function appendApplets(applets) { el.classList.add('loaded') break default: - console.log('react', el, payload) appendReactApplet(el, payload) el.classList.add('loaded') break diff --git a/client/map/index.js b/client/map/index.js index 788894f9..053cf13b 100644 --- a/client/map/index.js +++ b/client/map/index.js @@ -34,6 +34,7 @@ function addMarker(map, latlng, title, subtext) { subtext, ].join('')) } + function addArc(map, src, dest) { L.bezier({ path: [ @@ -46,14 +47,15 @@ function addArc(map, src, dest) { } export default function append(el, payload) { - const { cmd, data } = payload + const { data } = payload + let { paper, address } = data + let source = [0, 0] const citations = getCitations(data) - // console.log(el) let map = L.map(el).setView([25, 0], 2) L.tileLayer('https://api.tiles.mapbox.com/v4/{id}/{z}/{x}/{y}.png?access_token={accessToken}', { - attribution: 'Map data © <a href="https://www.openstreetmap.org/">OpenStreetMap</a> contributors,' + - '<a href="https://creativecommons.org/licenses/by-sa/2.0/">CC-BY-SA</a>,' + + attribution: 'Map data © <a href="https://www.openstreetmap.org/">OpenStreetMap</a> contributors, ' + + '<a href="https://creativecommons.org/licenses/by-sa/2.0/">CC-BY-SA</a>, ' + 'Imagery © <a href="https://www.mapbox.com/">Mapbox</a>', maxZoom: 18, id: 'mapbox.dark', @@ -61,13 +63,8 @@ export default function append(el, payload) { accessToken: 'pk.eyJ1IjoiZmFuc2FsY3kiLCJhIjoiY2pvN3I1czJwMHF5NDNrbWRoMWpteHlrdCJ9.kMpM5syQUhVjKkn1iVx9fg' }).addTo(map) - let { address } = data - console.log(address) - let source = [0, 0] if (address) { source = address.slice(3, 5).map(n => parseFloat(n)) - // console.log(map, address, source) - console.log(source) } citations.forEach(point => { @@ -77,5 +74,5 @@ export default function append(el, payload) { addArc(map, source, latlng) }) - addMarker(map, source, document.querySelector('h2').innerText, address[0]) + addMarker(map, source, paper.title, paper.address) } diff --git a/client/nameSearch/index.js b/client/nameSearch/index.js new file mode 100644 index 00000000..8c6475e4 --- /dev/null +++ b/client/nameSearch/index.js @@ -0,0 +1,5 @@ +import Container from './nameSearch.container' + +export { + Container, +} diff --git a/client/nameSearch/nameSearch.actions.js b/client/nameSearch/nameSearch.actions.js new file mode 100644 index 00000000..290ee38d --- /dev/null +++ b/client/nameSearch/nameSearch.actions.js @@ -0,0 +1,52 @@ +// import fetchJsonp from 'fetch-jsonp' +import * as types from '../types' +// import { hashPath } from '../util' +import { post } from '../util' +// import querystring from 'query-string' + +// urls + +const url = { + search: (dataset, q) => process.env.API_HOST + '/api/dataset/' + dataset + '/name?q=' + encodeURIComponent(q), +} +export const publicUrl = { +} + +// standard loading events + +const loading = (tag, offset) => ({ + type: types.nameSearch.loading, + tag, + offset +}) +const loaded = (tag, data, offset = 0) => ({ + type: types.nameSearch.loaded, + tag, + data, + offset +}) +const error = (tag, err) => ({ + type: types.nameSearch.error, + tag, + err +}) + +// search UI functions + +export const updateOptions = opt => dispatch => { + dispatch({ type: types.nameSearch.update_options, opt }) +} + +// API functions + +export const search = (payload, q) => dispatch => { + const tag = 'result' + const fd = new FormData() + fd.append('q', q) + dispatch(loading(tag)) + post(url.search(payload.dataset, q), fd) + .then(data => { + dispatch(loaded(tag, data)) + }) + .catch(err => dispatch(error(tag, err))) +} diff --git a/client/nameSearch/nameSearch.container.js b/client/nameSearch/nameSearch.container.js new file mode 100644 index 00000000..b0de0c3a --- /dev/null +++ b/client/nameSearch/nameSearch.container.js @@ -0,0 +1,24 @@ +import React, { Component } from 'react' +import { bindActionCreators } from 'redux' +import { connect } from 'react-redux' + +import * as actions from './nameSearch.actions' + +import NameSearchQuery from './nameSearch.query' +import NameSearchResult from './nameSearch.result' + +class NameSearchContainer extends Component { + render() { + const { payload } = this.props + // console.log(payload) + return ( + <div className='searchContainer'> + <NameSearchQuery payload={payload} /> + <NameSearchResult payload={payload} /> + </div> + ) + } +} + + +export default NameSearchContainer diff --git a/client/nameSearch/nameSearch.query.js b/client/nameSearch/nameSearch.query.js new file mode 100644 index 00000000..b82e324b --- /dev/null +++ b/client/nameSearch/nameSearch.query.js @@ -0,0 +1,48 @@ +import React, { Component } from 'react' +import { bindActionCreators } from 'redux' +import { connect } from 'react-redux' + +import * as actions from './nameSearch.actions' + +class NameSearchQuery extends Component { + state = { + value: null + } + + handleInput(value) { + this.setState({ q: value }) + if (value.length > 2) { + this.props.actions.search(this.props.payload, value) + } + } + + render() { + return ( + <div className='query'> + <h2>Find Your Name</h2> + <h3>Searching {13456} identities</h3> + <p> + {'Enter your name to see if you were included in this dataset..'} + </p> + <input + type="text" + class="q" + placeholder="Enter your name" + value={this.state.q} + onInput={e => this.handleInput(e.target.value)} + /> + </div> + ) + } +} + +const mapStateToProps = state => ({ + result: state.nameSearch.result, + options: state.nameSearch.options, +}) + +const mapDispatchToProps = dispatch => ({ + actions: bindActionCreators({ ...actions }, dispatch), +}) + +export default connect(mapStateToProps, mapDispatchToProps)(NameSearchQuery) diff --git a/client/nameSearch/nameSearch.reducer.js b/client/nameSearch/nameSearch.reducer.js new file mode 100644 index 00000000..101c93ea --- /dev/null +++ b/client/nameSearch/nameSearch.reducer.js @@ -0,0 +1,32 @@ +import * as types from '../types' + +const initialState = () => ({ + query: {}, + result: {}, + loading: false, +}) + +export default function nameSearchReducer(state = initialState(), action) { + switch (action.type) { + case types.nameSearch.loading: + return { + ...state, + [action.tag]: { loading: true }, + } + + case types.nameSearch.loaded: + return { + ...state, + [action.tag]: action.data, + } + + case types.nameSearch.error: + return { + ...state, + [action.tag]: { error: action.err }, + } + + default: + return state + } +} diff --git a/client/nameSearch/nameSearch.result.js b/client/nameSearch/nameSearch.result.js new file mode 100644 index 00000000..9e20228c --- /dev/null +++ b/client/nameSearch/nameSearch.result.js @@ -0,0 +1,88 @@ +import React, { Component } from 'react' +import { bindActionCreators } from 'redux' +import { connect } from 'react-redux' +import { courtesyS } from '../util' + +import * as actions from './nameSearch.actions' +import { Loader } from '../common' + +const errors = { + nomatch: ( + <div> + <h3>Name not found</h3> + {"No names matched your query."} + </div> + ), + error: ( + <div> + <h3>{"No matches found"}</h3> + </div> + ), +} + +class NameSearchResult extends Component { + render() { + const { dataset } = this.props.payload + const { query, results, loading, error } = this.props.result + console.log(this.props.result) + if (loading) { + return ( + <div className='result'> + <div> + <Loader /> + </div> + </div> + ) + } + if (error) { + console.log(error) + let errorMessage = errors[error] || errors.error + return ( + <div className='result'>{errorMessage}</div> + ) + } + if (!results) { + return <div className='result'></div> + } + if (!results.length) { + return ( + <div className='result'>{errors.nomatch}</div> + ) + } + const els = results.map((result, i) => { + const { uuid } = result.uuid + const { fullname, gender, description, images } = result.identity + return ( + <div key={i}> + <img src={'https://megapixels.nyc3.digitaloceanspaces.com/v1/media/' + dataset + '/' + uuid + '.jpg'} /> + {fullname} {'('}{gender}{')'}<br/> + {description}<br/> + {courtesyS(images, 'image')}{' in dataset'}<br /> + </div> + ) + }) + + return ( + <div className='result'> + <div className="timing"> + {'Search took '}{Math.round(query.timing * 1000) + ' ms'} + </div> + <div className='results'> + {els} + </div> + </div> + ) + } +} + +const mapStateToProps = state => ({ + query: state.nameSearch.query, + result: state.nameSearch.result, + options: state.nameSearch.options, +}) + +const mapDispatchToProps = dispatch => ({ + actions: bindActionCreators({ ...actions }, dispatch), +}) + +export default connect(mapStateToProps, mapDispatchToProps)(NameSearchResult) diff --git a/client/store.js b/client/store.js index 03f983a5..13612f2d 100644 --- a/client/store.js +++ b/client/store.js @@ -1,16 +1,12 @@ import { applyMiddleware, compose, combineReducers, createStore } from 'redux' import thunk from 'redux-thunk' -// import metadataReducer from './metadata/metadata.reducer' import faceSearchReducer from './faceSearch/faceSearch.reducer' -// import reviewReducer from './review/review.reducer' +import nameSearchReducer from './nameSearch/nameSearch.reducer' const rootReducer = combineReducers({ - auth: (state = {}) => state, - // auth: (state = login()) => state, - // metadata: metadataReducer, faceSearch: faceSearchReducer, - // review: reviewReducer, + nameSearch: nameSearchReducer, }) function configureStore(initialState = {}) { diff --git a/client/tables.js b/client/tables.js index 2a2699f9..b4c13887 100644 --- a/client/tables.js +++ b/client/tables.js @@ -74,4 +74,8 @@ export default function append(el, payload) { } }) } + + if (fields.length > 1 && fields[1].indexOf('filter')) { + const filter = fields[1].split(' ') + } } diff --git a/client/types.js b/client/types.js index d295d0d1..fb1fbe30 100644 --- a/client/types.js +++ b/client/types.js @@ -10,5 +10,8 @@ export const faceSearch = tagAsType('faceSearch', [ 'loading', 'loaded', 'error', 'update_options', ]) +export const nameSearch = tagAsType('nameSearch', [ + 'loading', 'loaded', 'error', 'update_options', +]) export const init = '@@INIT' diff --git a/megapixels/app/models/sql_factory.py b/megapixels/app/models/sql_factory.py index 9a44941b..da95b539 100644 --- a/megapixels/app/models/sql_factory.py +++ b/megapixels/app/models/sql_factory.py @@ -10,7 +10,7 @@ from sqlalchemy.ext.declarative import declarative_base from app.utils.file_utils import load_recipe, load_csv_safe from app.settings import app_cfg as cfg -connection_url = "mysql+mysqldb://{}:{}@{}/{}".format( +connection_url = "mysql+mysqlconnector://{}:{}@{}/{}?charset=utf8mb4".format( os.getenv("DB_USER"), os.getenv("DB_PASS"), os.getenv("DB_HOST"), @@ -35,7 +35,12 @@ def load_sql_datasets(replace=False, base_model=None): global datasets, loaded, Session if loaded: return datasets - engine = create_engine(connection_url) + engine = create_engine(connection_url, encoding="utf-8") + # db.set_character_set('utf8') + # dbc = db.cursor() + # dbc.execute('SET NAMES utf8;') + # dbc.execute('SET CHARACTER SET utf8;') + # dbc.execute('SET character_set_connection=utf8;') Session = sessionmaker(bind=engine) for path in glob.iglob(os.path.join(cfg.DIR_FAISS_METADATA, "*")): dataset = load_sql_dataset(path, replace, engine, base_model) @@ -92,13 +97,27 @@ class SqlDataset: 'pose': self.select('pose', id), } + def search_name(self, q): + table = self.get_table('identity_meta') + uuid_table = self.get_table('uuids') + + identity = table.query.filter(table.fullname.like(q)).order_by(table.fullname.desc()).limit(30) + identities = [] + for row in identity: + uuid = uuid_table.query.filter(uuid_table.id == row.image_id).first() + identities.append({ + 'uuid': uuid.toJSON(), + 'identity': row.toJSON(), + }) + return identities + def select(self, table, id): table = self.get_table(table) if not table: return None session = Session() # for obj in session.query(table).filter_by(id=id): - print(table) + # print(table) obj = session.query(table).filter(table.id == id).first() session.close() return obj.toJSON() @@ -125,7 +144,7 @@ class SqlDataset: class UUID(self.base_model): __tablename__ = self.name + "_uuid" id = Column(Integer, primary_key=True) - uuid = Column(String(36), nullable=False) + uuid = Column(String(36, convert_unicode=True), nullable=False) def toJSON(self): return { 'id': self.id, @@ -167,9 +186,9 @@ class SqlDataset: class Identity(self.base_model): __tablename__ = self.name + "_identity" id = Column(Integer, primary_key=True) - fullname = Column(String(36), nullable=False) - description = Column(String(36), nullable=False) - gender = Column(String(1), nullable=False) + fullname = Column(String(36, convert_unicode=True), nullable=False) + description = Column(String(36, convert_unicode=True), nullable=False) + gender = Column(String(1, convert_unicode=True), nullable=False) images = Column(Integer, nullable=False) image_id = Column(Integer, nullable=False) def toJSON(self): diff --git a/megapixels/app/server/api.py b/megapixels/app/server/api.py index bc60118c..35862837 100644 --- a/megapixels/app/server/api.py +++ b/megapixels/app/server/api.py @@ -15,68 +15,84 @@ from app.utils.im_utils import pil2np sanitize_re = re.compile('[\W]+') valid_exts = ['.gif', '.jpg', '.jpeg', '.png'] +LIMIT = 9 +THRESHOLD = 0.3 + api = Blueprint('api', __name__) faiss_datasets = load_faiss_databases() @api.route('/') def index(): + """List the datasets and their fields""" return jsonify({ 'datasets': list_datasets() }) -@api.route('/dataset/<name>') -def show(name): - dataset = get_dataset(name) + +@api.route('/dataset/<dataset_name>') +def show(dataset_name): + """Show the data that a dataset will return""" + dataset = get_dataset(dataset_name) if dataset: return jsonify(dataset.describe()) else: return jsonify({ 'status': 404 }) -@api.route('/dataset/<name>/face/', methods=['POST']) -def upload(name): + +@api.route('/dataset/<dataset_name>/face', methods=['POST']) +def upload(dataset_name): + """Query an image against FAISS and return the matching identities""" start = time.time() - dataset = get_dataset(name) - if name not in faiss_datasets: + dataset = get_dataset(dataset_name) + if dataset_name not in faiss_datasets: return jsonify({ 'error': 'invalid dataset' }) - faiss_dataset = faiss_datasets[name] + faiss_dataset = faiss_datasets[dataset_name] file = request.files['query_img'] fn = file.filename if fn.endswith('blob'): fn = 'filename.jpg' basename, ext = os.path.splitext(fn) - print("got {}, type {}".format(basename, ext)) + # print("got {}, type {}".format(basename, ext)) if ext.lower() not in valid_exts: return jsonify({ 'error': 'not an image' }) im = Image.open(file.stream).convert('RGB') im_np = pil2np(im) - + # Face detection detector = face_detector.DetectorDLIBHOG() # get detection as BBox object bboxes = detector.detect(im_np, largest=True) - if not len(bboxes): + if not bboxes or not len(bboxes): return jsonify({ 'error': 'bbox' }) bbox = bboxes[0] + if not bbox: + return jsonify({ + 'error': 'bbox' + }) + dim = im_np.shape[:2][::-1] bbox = bbox.to_dim(dim) # convert back to real dimensions + # print("got bbox") + if not bbox: + return jsonify({ + 'error': 'bbox' + }) - # face recognition/vector + # extract 128-D vector recognition = face_recognition.RecognitionDLIB(gpu=-1) vec = recognition.vec(im_np, bbox) - - # print(vec) query = np.array([ vec ]).astype('float32') - # query FAISS! - distances, indexes = faiss_dataset.search(query, 10) + # query FAISS + distances, indexes = faiss_dataset.search(query, LIMIT) - if len(indexes) == 0: + if len(indexes) == 0 or len(indexes[0]) == 0: return jsonify({ 'error': 'nomatch' }) @@ -85,48 +101,51 @@ def upload(name): distances = distances[0] indexes = indexes[0] - if len(indexes) == 0: - return jsonify({ - 'error': 'nomatch' - }) - - lookup = {} - ids = [i+1 for i in indexes] + dists = [] + ids = [] for _d, _i in zip(distances, indexes): - lookup[_i+1] = _d + if _d <= THRESHOLD: + dists.append(round(float(_d), 2)) + ids.append(_i+1) + + results = [ dataset.get_identity(int(_i)) for _i in ids ] - print(distances) - print(indexes) + # print(distances) + # print(ids) - # with the result we have an ID - # query the sql dataset for the UUID etc here + # 'bbox': str(bboxes[0]), + # 'bbox_dim': str(bbox), + # print(bboxes[0]) + # print(bbox) query = { - 'timing': time.time() - start, + 'timing': round(time.time() - start, 3), + 'bbox': str(bbox), } - results = [ dataset.get_identity(id) for id in ids ] - - print(results) + # print(results) return jsonify({ + 'query': query, 'results': results, - 'distances': distances.tolist(), - 'indexes': indexes.tolist(), + 'distances': dists, }) -@api.route('/dataset/<name>/name', methods=['GET']) -def name_lookup(dataset): + +@api.route('/dataset/<dataset_name>/name', methods=['GET','POST']) +def name_lookup(dataset_name): + """Find a name in the dataset""" start = time.time() - dataset = get_dataset(name) + dataset = get_dataset(dataset_name) - # we have a query from the request query string... - # use this to do a like* query on the identities_meta table + q = request.args.get('q') + # print(q) query = { + 'q': q, 'timing': time.time() - start, } - results = [] - - print(results) + results = dataset.search_name(q + '%') if q else None + + # print(results) return jsonify({ 'query': query, 'results': results, diff --git a/megapixels/app/settings/app_cfg.py b/megapixels/app/settings/app_cfg.py index 0c28b315..55fed166 100644 --- a/megapixels/app/settings/app_cfg.py +++ b/megapixels/app/settings/app_cfg.py @@ -7,6 +7,8 @@ from dotenv import load_dotenv from app.settings import types from app.utils import click_utils +import codecs +codecs.register(lambda name: codecs.lookup('utf8') if name == 'utf8mb4' else None) # ----------------------------------------------------------------------------- # Enun lists used for custom Click Params @@ -87,7 +89,7 @@ CKPT_ZERO_PADDING = 9 HASH_TREE_DEPTH = 3 HASH_BRANCH_SIZE = 3 -DLIB_FACEREC_JITTERS = 25 # number of face recognition jitters +DLIB_FACEREC_JITTERS = 5 # number of face recognition jitters DLIB_FACEREC_PADDING = 0.25 # default dlib POSE_MINMAX_YAW = (-25,25) diff --git a/megapixels/app/site/parser.py b/megapixels/app/site/parser.py index ecfae0cb..b3d3a8c2 100644 --- a/megapixels/app/site/parser.py +++ b/megapixels/app/site/parser.py @@ -64,7 +64,7 @@ def format_applet(section, s3_path): else: command = payload[0] opt = None - if command == 'python': + if command == 'python' or command == 'javascript' or command == 'code': return format_section([ section ], s3_path) applet['command'] = command diff --git a/site/assets/css/applets.css b/site/assets/css/applets.css index ecba518c..315d72e0 100644 --- a/site/assets/css/applets.css +++ b/site/assets/css/applets.css @@ -16,8 +16,18 @@ flex-direction: row; justify-content: flex-start; } - +.q { + width: 100%; + padding: 5px; + font-size: 14pt; +} +.timing { + font-size: 9pt; + padding-top: 10px; +} .results { + margin-top: 10px; + padding-bottom: 10px; display: flex; flex-direction: row; flex-wrap: wrap; @@ -27,11 +37,15 @@ margin-left: 20px; margin-bottom: 40px; font-size: 8pt; + background: #000; + padding: 5px; + font-weight: 500; } .results > div img { + display: block; margin-bottom: 4px; - width: 200px; - height: 200px; + width: 190px; + height: 190px; background: rgba(255,255,255,0.05); } .results > div:nth-child(3n+1) { @@ -40,6 +54,15 @@ .query h2 { margin-top: 0; padding-top: 0; } +.img { + position: relative; +} +.img .bbox { + position: absolute; + color: rgba(255,0,0,1); + background: rgba(255,0,0,0.05); + border: 1px solid; +} .cta { padding-left: 20px; font-size: 11pt; diff --git a/site/assets/css/css.css b/site/assets/css/css.css index 4f2d7c6e..003ac4a3 100644 --- a/site/assets/css/css.css +++ b/site/assets/css/css.css @@ -27,6 +27,7 @@ header { left: 0; width: 100%; height: 70px; + z-index: 1; background: #1e1e1e; display: flex; flex-direction: row; diff --git a/site/public/test/index.html b/site/public/test/index.html index b4d16036..41f8eda5 100644 --- a/site/public/test/index.html +++ b/site/public/test/index.html @@ -30,14 +30,14 @@ <section><h1>Megapixels UI Tests</h1> <ul> -<li><a href="/test/style">Style Guide</a></li> -<li><a href="/test/csv">CSV</a></li> -<li><a href="/test/datasets/">Dataset list</a></li> -<li><a href="/test/citations/">Citation list</a></li> -<li><a href="/test/map/">Citation map</a></li> -<li><a href="/test/face_search/">Face search</a></li> -<li><a href="/test/name_search/">Name search</a></li> -<li><a href="/test/gallery/">Modal image gallery</a></li> +<li><a href="/test/style/index.html">Style Guide</a></li> +<li><a href="/test/csv/index.html">CSV</a></li> +<li><a href="/test/datasets/index.html">Dataset list</a></li> +<li><a href="/test/citations/index.html">Citation list</a></li> +<li><a href="/test/map/index.html">Citation map</a></li> +<li><a href="/test/face_search/index.html">Face search</a></li> +<li><a href="/test/name_search/index.html">Name search</a></li> +<li><a href="/test/gallery/index.html">Modal image gallery</a></li> </ul> </section> diff --git a/site/public/test/style/index.html b/site/public/test/style/index.html index 6d99a236..ab13a589 100644 --- a/site/public/test/style/index.html +++ b/site/public/test/style/index.html @@ -54,10 +54,16 @@ <div class='image'><img src='https://nyc3.digitaloceanspaces.com/megapixels/v1/site/test/assets/man.jpg' alt='Person 3. Let me tell you about Person 3. This person has a very long description with text which wraps like crazy'><div class='caption'>Person 3. Let me tell you about Person 3. This person has a very long description with text which wraps like crazy</div></div></section><section><blockquote><p>est, qui dolorem ipsum, quia dolor sit amet consectetur adipisci[ng] velit, sed quia non-numquam [do] eius modi tempora inci[di]dunt, ut labore et dolore magnam aliquam quaerat voluptatem.</p> </blockquote> </section><section class='wide'><div class='image'><img src='https://nyc3.digitaloceanspaces.com/megapixels/v1/site/test/assets/wide-test.jpg' alt='This image is extremely wide and the text beneath it will wrap but thats fine because it can also contain <a href="https://example.com/">hyperlinks</a>! Yes, you read that right—hyperlinks! Lorem ipsum dolor sit amet ad volotesque sic hoc ad nauseam'><div class='caption'>This image is extremely wide and the text beneath it will wrap but that's fine because it can also contain <a href="https://example.com/">hyperlinks</a>! Yes, you read that right—hyperlinks! Lorem ipsum dolor sit amet ad volotesque sic hoc ad nauseam</div></div></section><section><p>Inline <code>code</code> has <code>back-ticks around</code> it.</p> -</section><section class='applet_container'><div class='applet' data-payload='{"command": "javascript", "fields": ["var s = \"JavaScript syntax highlighting\";", "alert(s);"]}'></div></section><section><pre><code class="lang-python">s = "Python syntax highlighting" +</section><section><pre><code class="lang-javascript">var s = "JavaScript syntax highlighting"; +alert(s); +</code></pre> +</section><section><pre><code class="lang-python">s = "Python syntax highlighting" print(s) </code></pre> -</section><section class='applet_container'><div class='applet' data-payload='{"command": "No language indicated, so no syntax highlighting. ", "fields": ["But let's throw in a <b>tag</b>."]}'></div></section><section><p>Horizontal rule</p> +</section><section><pre><code class="lang-code">Generic code block. Note that code blocks that are not so marked will not appear. +But let's throw in a <b>tag</b>. +</code></pre> +</section><section><p>Horizontal rule</p> <hr> <p>Citations below here</p> <div class="footnotes"> diff --git a/webpack.config.dev.js b/webpack.config.dev.js index d6f7af46..4137a948 100644 --- a/webpack.config.dev.js +++ b/webpack.config.dev.js @@ -1,6 +1,6 @@ require('dotenv').config() -const HtmlWebpackPlugin = require('html-webpack-plugin') +// const HtmlWebpackPlugin = require('html-webpack-plugin') // const CleanWebpackPlugin = require('clean-webpack-plugin') const webpack = require('webpack') const path = require('path') @@ -13,19 +13,9 @@ module.exports = { path: path.resolve(__dirname, 'site/assets/js/dist'), filename: 'index.js' }, - // devServer: { - // port: 9000, - // headers: { - // 'Access-Control-Allow-Origin': '*', - // }, - // publicPath: '/site/assets/js/dist/', - // hot: true, - // }, devtool: 'inline-source-map', resolve: { alias: { - // 'vcat-header': path.resolve(__dirname, '../app/components/common/header.component.js'), - // 'vcat-auth-reducer': path.resolve(__dirname, '../app/reducers/auth.reducer.js'), "react": "preact-compat", "react-dom": "preact-compat" } @@ -35,7 +25,6 @@ module.exports = { new webpack.DefinePlugin({ 'process.env.NODE_ENV': '"development"', 'process.env.S3_HOST': '"' + process.env.S3_HOST + '"', - // 'process.env.VCAT_HOST': '"http://127.0.0.1:8000"', 'process.env.API_HOST': '""', }), // new HtmlWebpackPlugin({ diff --git a/webpack.config.prod.js b/webpack.config.prod.js index b9d3f411..f5da2bb3 100644 --- a/webpack.config.prod.js +++ b/webpack.config.prod.js @@ -17,8 +17,7 @@ module.exports = { new webpack.DefinePlugin({ 'process.env.NODE_ENV': '"production"', 'process.env.S3_HOST': '"' + process.env.S3_HOST + '"', - // 'process.env.VCAT_HOST': '""', - // 'process.env.API_HOST': '"https://syrianarchive.vframe.io"', + 'process.env.API_HOST': '""', }), new UglifyJsPlugin(), new webpack.optimize.AggressiveMergingPlugin() |