diff options
Diffstat (limited to 'megapixels/commands/datasets/megaface_flickr_api.py')
| -rw-r--r-- | megapixels/commands/datasets/megaface_flickr_api.py | 141 |
1 files changed, 0 insertions, 141 deletions
diff --git a/megapixels/commands/datasets/megaface_flickr_api.py b/megapixels/commands/datasets/megaface_flickr_api.py deleted file mode 100644 index 62232ab8..00000000 --- a/megapixels/commands/datasets/megaface_flickr_api.py +++ /dev/null @@ -1,141 +0,0 @@ -from glob import glob -import os -from os.path import join -from pathlib import Path - -import click - -from app.settings import types -from app.utils import click_utils -from app.settings import app_cfg as cfg -from app.utils import logger_utils - -import dlib -import pandas as pd -from PIL import Image, ImageOps, ImageFilter -from app.utils import file_utils, im_utils - - -log = logger_utils.Logger.getLogger() - -@click.command() -@click.option('-i', '--input', 'opt_fp_in', required=True, - help='Input directory') -@click.option('-o', '--output', 'opt_fp_out', - help='Output directory') -@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None), - help='Slice list of files') -@click.option('-d', '--delay', 'opt_delay', default=None, type=int, - help='Delay between API calls to prevent rate-limiting') -@click.option('--checkpoints', 'opt_checkpoints', is_flag=True, - help='Save checkpoints') -@click.option('--api_key', 'opt_api_key', envvar='FLICKR_API_KEY') -@click.option('--api_secret', 'opt_api_secret', envvar='FLICKR_API_SECRET') -@click.option('--checkpoint_interval', 'opt_ckpt_interval', default=10000, - help='Save checkpoint interval') -@click.pass_context -def cli(ctx, opt_fp_in, opt_fp_out, opt_slice, opt_api_key, opt_api_secret, - opt_delay, opt_checkpoints, opt_ckpt_interval): - """Appends Flickr API info to CSV""" - - from tqdm import tqdm - from glob import glob - import time - import flickr_api # pip install flickr_api - from flickr_api.flickrerrors import FlickrAPIError - - # ------------------------------------------------- - # process - - if not opt_api_key or not opt_api_secret: - log.error('source .env vars for Flickr API and try again') - return - - # init Flickr API - flickr_api.set_keys(api_key=opt_api_key, api_secret=opt_api_secret) - - # reqd in CSV - df_ids = pd.read_csv(opt_fp_in) - if opt_slice: - df_ids = df_ids[opt_slice[0]:opt_slice[1]] - - log.info('Processing: {:,} items'.format(len(df_ids))) - - # iterate MegaFace IDs - identities = [] - - tqdm.pandas() - - for idx, df_id in tqdm(df_ids.iterrows(), total=len(df_ids)): - # a = flickr_api.Person(id='123456789@N01') - df_id_dict = dict(df_id) - - # append relevant data - try: - person = flickr_api.Person(id=df_id['nsid']) - info = person.getInfo() - df_id_dict.update( { - 'user_name': info.get('username', ''), - 'location': info.get('location', ''), - 'real_name': info.get('realname', ''), - 'time_zone': info.get('timezone', {}).get('timezone_id', ''), - 'time_first_photo': info.get('photos_info', {}).get('firstdatetaken'), - 'photos_count': info.get('photos_info', {}).get('count'), - 'description': info.get('description', ''), - 'id': info.get('id'), - 'path_alias': info.get('path_alias', ''), - 'is_pro': info.get('ispro', ''), - 'url_photos': info.get('photosurl', ''), - 'url_profile': info.get('photosurl', ''), - 'url_mobile': info.get('mobileurl', ''), - }) - identities.append(df_id_dict) - - except FlickrAPIError as e: - log.error(e) - - - if opt_checkpoints: - if (idx + 1) % opt_ckpt_interval == 0: - df = pd.DataFrame.from_dict(identities) - fpp_out = Path(opt_fp_out) - opt_fp_out_ckpt = join(fpp_out.parent, '{}_ckpt_{}.csv'.format(fpp_out.stem, file_utils.zpad(idx + 1))) - log.info('Saving checkpoint {:,} to {}'.format(idx + 1, opt_fp_out_ckpt)) - df.to_csv(opt_fp_out_ckpt, index=False) - - if opt_delay: - time.sleep(opt_delay) - - - df = pd.DataFrame.from_dict(identities) - df.to_csv(opt_fp_out, index=False) - - log.info('Wrote: {:,} lines to {}'.format(len(df), opt_fp_out)) - - -""" -Example API data: -{'id': '7124086@N07', - 'nsid': '7124086@N07', - 'ispro': 1, - 'can_buy_pro': 0, - 'iconserver': '2325', - 'iconfarm': 3, - 'path_alias': 'shirleylin', - 'has_stats': '1', - 'pro_badge': 'standard', - 'expire': '0', - 'username': 'ShirleyLin', - 'realname': 'Shirley Lin', - 'location': 'Fremont, California, US', - 'timezone': {'label': 'Pacific Time (US & Canada); Tijuana', - 'offset': '-08:00', - 'timezone_id': 'PST8PDT'}, - 'description': '', - 'photosurl': 'https://www.flickr.com/photos/shirleylin/', - 'profileurl': 'https://www.flickr.com/people/shirleylin/', - 'mobileurl': 'https://m.flickr.com/photostream.gne?id=7102756', - 'photos_info': {'firstdatetaken': '2004-05-24 12:12:15', - 'firstdate': '1172556588', - 'count': 9665}} -"""
\ No newline at end of file |