diff options
Diffstat (limited to 'megapixels/commands/datasets/megaface_flickr_api.py')
| -rw-r--r-- | megapixels/commands/datasets/megaface_flickr_api.py | 141 |
1 files changed, 141 insertions, 0 deletions
diff --git a/megapixels/commands/datasets/megaface_flickr_api.py b/megapixels/commands/datasets/megaface_flickr_api.py new file mode 100644 index 00000000..62232ab8 --- /dev/null +++ b/megapixels/commands/datasets/megaface_flickr_api.py @@ -0,0 +1,141 @@ +from glob import glob +import os +from os.path import join +from pathlib import Path + +import click + +from app.settings import types +from app.utils import click_utils +from app.settings import app_cfg as cfg +from app.utils import logger_utils + +import dlib +import pandas as pd +from PIL import Image, ImageOps, ImageFilter +from app.utils import file_utils, im_utils + + +log = logger_utils.Logger.getLogger() + +@click.command() +@click.option('-i', '--input', 'opt_fp_in', required=True, + help='Input directory') +@click.option('-o', '--output', 'opt_fp_out', + help='Output directory') +@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None), + help='Slice list of files') +@click.option('-d', '--delay', 'opt_delay', default=None, type=int, + help='Delay between API calls to prevent rate-limiting') +@click.option('--checkpoints', 'opt_checkpoints', is_flag=True, + help='Save checkpoints') +@click.option('--api_key', 'opt_api_key', envvar='FLICKR_API_KEY') +@click.option('--api_secret', 'opt_api_secret', envvar='FLICKR_API_SECRET') +@click.option('--checkpoint_interval', 'opt_ckpt_interval', default=10000, + help='Save checkpoint interval') +@click.pass_context +def cli(ctx, opt_fp_in, opt_fp_out, opt_slice, opt_api_key, opt_api_secret, + opt_delay, opt_checkpoints, opt_ckpt_interval): + """Appends Flickr API info to CSV""" + + from tqdm import tqdm + from glob import glob + import time + import flickr_api # pip install flickr_api + from flickr_api.flickrerrors import FlickrAPIError + + # ------------------------------------------------- + # process + + if not opt_api_key or not opt_api_secret: + log.error('source .env vars for Flickr API and try again') + return + + # init Flickr API + flickr_api.set_keys(api_key=opt_api_key, api_secret=opt_api_secret) + + # reqd in CSV + df_ids = pd.read_csv(opt_fp_in) + if opt_slice: + df_ids = df_ids[opt_slice[0]:opt_slice[1]] + + log.info('Processing: {:,} items'.format(len(df_ids))) + + # iterate MegaFace IDs + identities = [] + + tqdm.pandas() + + for idx, df_id in tqdm(df_ids.iterrows(), total=len(df_ids)): + # a = flickr_api.Person(id='123456789@N01') + df_id_dict = dict(df_id) + + # append relevant data + try: + person = flickr_api.Person(id=df_id['nsid']) + info = person.getInfo() + df_id_dict.update( { + 'user_name': info.get('username', ''), + 'location': info.get('location', ''), + 'real_name': info.get('realname', ''), + 'time_zone': info.get('timezone', {}).get('timezone_id', ''), + 'time_first_photo': info.get('photos_info', {}).get('firstdatetaken'), + 'photos_count': info.get('photos_info', {}).get('count'), + 'description': info.get('description', ''), + 'id': info.get('id'), + 'path_alias': info.get('path_alias', ''), + 'is_pro': info.get('ispro', ''), + 'url_photos': info.get('photosurl', ''), + 'url_profile': info.get('photosurl', ''), + 'url_mobile': info.get('mobileurl', ''), + }) + identities.append(df_id_dict) + + except FlickrAPIError as e: + log.error(e) + + + if opt_checkpoints: + if (idx + 1) % opt_ckpt_interval == 0: + df = pd.DataFrame.from_dict(identities) + fpp_out = Path(opt_fp_out) + opt_fp_out_ckpt = join(fpp_out.parent, '{}_ckpt_{}.csv'.format(fpp_out.stem, file_utils.zpad(idx + 1))) + log.info('Saving checkpoint {:,} to {}'.format(idx + 1, opt_fp_out_ckpt)) + df.to_csv(opt_fp_out_ckpt, index=False) + + if opt_delay: + time.sleep(opt_delay) + + + df = pd.DataFrame.from_dict(identities) + df.to_csv(opt_fp_out, index=False) + + log.info('Wrote: {:,} lines to {}'.format(len(df), opt_fp_out)) + + +""" +Example API data: +{'id': '7124086@N07', + 'nsid': '7124086@N07', + 'ispro': 1, + 'can_buy_pro': 0, + 'iconserver': '2325', + 'iconfarm': 3, + 'path_alias': 'shirleylin', + 'has_stats': '1', + 'pro_badge': 'standard', + 'expire': '0', + 'username': 'ShirleyLin', + 'realname': 'Shirley Lin', + 'location': 'Fremont, California, US', + 'timezone': {'label': 'Pacific Time (US & Canada); Tijuana', + 'offset': '-08:00', + 'timezone_id': 'PST8PDT'}, + 'description': '', + 'photosurl': 'https://www.flickr.com/photos/shirleylin/', + 'profileurl': 'https://www.flickr.com/people/shirleylin/', + 'mobileurl': 'https://m.flickr.com/photostream.gne?id=7102756', + 'photos_info': {'firstdatetaken': '2004-05-24 12:12:15', + 'firstdate': '1172556588', + 'count': 9665}} +"""
\ No newline at end of file |