.

1 files changed, 141 insertions, 0 deletions

diff --git a/megapixels/commands/datasets/megaface_flickr_api.py b/megapixels/commands/datasets/megaface_flickr_api.py
new file mode 100644
index 00000000..62232ab8
--- /dev/null
+++ b/megapixels/commands/datasets/megaface_flickr_api.py
@@ -0,0 +1,141 @@
+from glob import glob
+import os
+from os.path import join
+from pathlib import Path
+
+import click
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+from app.utils import logger_utils
+
+import dlib
+import pandas as pd
+from PIL import Image, ImageOps, ImageFilter
+from app.utils import file_utils, im_utils
+
+
+log = logger_utils.Logger.getLogger()
+
+@click.command()
+@click.option('-i', '--input', 'opt_fp_in', required=True,
+  help='Input directory')
+@click.option('-o', '--output', 'opt_fp_out',
+  help='Output directory')
+@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None),
+  help='Slice list of files')
+@click.option('-d', '--delay', 'opt_delay', default=None, type=int,
+  help='Delay between API calls to prevent rate-limiting')
+@click.option('--checkpoints', 'opt_checkpoints', is_flag=True,
+  help='Save checkpoints')
+@click.option('--api_key', 'opt_api_key', envvar='FLICKR_API_KEY')
+@click.option('--api_secret', 'opt_api_secret', envvar='FLICKR_API_SECRET')
+@click.option('--checkpoint_interval', 'opt_ckpt_interval', default=10000,
+  help='Save checkpoint interval')
+@click.pass_context
+def cli(ctx, opt_fp_in, opt_fp_out, opt_slice, opt_api_key, opt_api_secret, 
+  opt_delay, opt_checkpoints, opt_ckpt_interval):
+  """Appends Flickr API info to CSV"""
+  
+  from tqdm import tqdm
+  from glob import glob
+  import time
+  import flickr_api  # pip install flickr_api
+  from flickr_api.flickrerrors import FlickrAPIError
+  
+  # -------------------------------------------------
+  # process  
+  
+  if not opt_api_key or not opt_api_secret:
+    log.error('source .env vars for Flickr API and try again')
+    return
+
+  # init Flickr API
+  flickr_api.set_keys(api_key=opt_api_key, api_secret=opt_api_secret)
+
+  # reqd in CSV
+  df_ids = pd.read_csv(opt_fp_in)
+  if opt_slice:
+    df_ids = df_ids[opt_slice[0]:opt_slice[1]]
+  
+  log.info('Processing: {:,} items'.format(len(df_ids)))
+
+  # iterate MegaFace IDs
+  identities = []
+  
+  tqdm.pandas()
+
+  for idx, df_id in tqdm(df_ids.iterrows(), total=len(df_ids)):
+    # a = flickr_api.Person(id='123456789@N01')
+    df_id_dict = dict(df_id)
+    
+    # append relevant data
+    try:
+      person = flickr_api.Person(id=df_id['nsid'])
+      info = person.getInfo()
+      df_id_dict.update( {
+        'user_name': info.get('username', ''), 
+        'location': info.get('location', ''),
+        'real_name': info.get('realname', ''),
+        'time_zone': info.get('timezone', {}).get('timezone_id', ''),
+        'time_first_photo': info.get('photos_info', {}).get('firstdatetaken'),
+        'photos_count': info.get('photos_info', {}).get('count'),
+        'description': info.get('description', ''),
+        'id': info.get('id'),
+        'path_alias': info.get('path_alias', ''),
+        'is_pro': info.get('ispro', ''),
+        'url_photos': info.get('photosurl', ''),
+        'url_profile': info.get('photosurl', ''),
+        'url_mobile': info.get('mobileurl', ''),
+        })
+      identities.append(df_id_dict)
+
+    except FlickrAPIError as e:
+      log.error(e)
+      
+
+    if opt_checkpoints:
+      if (idx + 1) % opt_ckpt_interval == 0:
+        df = pd.DataFrame.from_dict(identities)
+        fpp_out = Path(opt_fp_out)
+        opt_fp_out_ckpt = join(fpp_out.parent, '{}_ckpt_{}.csv'.format(fpp_out.stem, file_utils.zpad(idx + 1)))
+        log.info('Saving checkpoint {:,} to {}'.format(idx + 1, opt_fp_out_ckpt))
+        df.to_csv(opt_fp_out_ckpt, index=False)
+
+    if opt_delay:
+      time.sleep(opt_delay)
+
+
+  df = pd.DataFrame.from_dict(identities)
+  df.to_csv(opt_fp_out, index=False)
+  
+  log.info('Wrote: {:,} lines to {}'.format(len(df), opt_fp_out))
+
+
+"""
+Example API data:
+{'id': '7124086@N07',
+ 'nsid': '7124086@N07',
+ 'ispro': 1,
+ 'can_buy_pro': 0,
+ 'iconserver': '2325',
+ 'iconfarm': 3,
+ 'path_alias': 'shirleylin',
+ 'has_stats': '1',
+ 'pro_badge': 'standard',
+ 'expire': '0',
+ 'username': 'ShirleyLin',
+ 'realname': 'Shirley Lin',
+ 'location': 'Fremont, California, US',
+ 'timezone': {'label': 'Pacific Time (US & Canada); Tijuana',
+  'offset': '-08:00',
+  'timezone_id': 'PST8PDT'},
+ 'description': '',
+ 'photosurl': 'https://www.flickr.com/photos/shirleylin/',
+ 'profileurl': 'https://www.flickr.com/people/shirleylin/',
+ 'mobileurl': 'https://m.flickr.com/photostream.gne?id=7102756',
+ 'photos_info': {'firstdatetaken': '2004-05-24 12:12:15',
+  'firstdate': '1172556588',
+  'count': 9665}}
+"""
\ No newline at end of file