{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# UMD Faces Knowledge Graph Identities\n",
    "\n",
    "- convert filename-names to names\n",
    "- fetch Google Knowledge Graph entity IDs for each name\n",
    "- save KG IDs to CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import os\n",
    "import os.path as osp\n",
    "from os.path import join\n",
    "from glob import glob\n",
    "import random\n",
    "import math\n",
    "from datetime import datetime\n",
    "import requests\n",
    "import json\n",
    "import time\n",
    "from pprint import pprint\n",
    "from multiprocessing.pool import ThreadPool\n",
    "import threading\n",
    "import urllib.request\n",
    "\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "import pandas as pd\n",
    "from scipy.io import loadmat\n",
    "import numpy as np\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load IMDB Metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "fp_filenames = '/data_store_hdd/datasets/people/umd_faces/downloads/filenames.txt'\n",
    "with open(fp_filenames, 'r') as fp:\n",
    "  filenames = fp.readlines()\n",
    "_ = filenames.pop(0)\n",
    "filenames = [x.replace('_', ' ').strip() for x in filenames]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "aaron rodgers\n"
     ]
    }
   ],
   "source": [
    "print(filenames[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Google Knowledge Graph API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read API key\n",
    "api_key = open('/work/megapixels_dev/3rdparty/knowledge-graph-api/.api_key').read()\n",
    "url_kg_api = 'https://kgsearch.googleapis.com/v1/entities:search'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _get_kg_meta(result_obj, params):\n",
    "  global api_key, url_kg_api\n",
    "  \n",
    "  params['indent'] = True\n",
    "  params['key'] = api_key\n",
    "  params['limit'] = 1\n",
    "  \n",
    "  url = f'{url_kg_api}?{urllib.parse.urlencode(params)}'\n",
    "  try:\n",
    "    json_response = urllib.request.urlopen(url).read()\n",
    "  except Exception as e:\n",
    "    result['error'] = str(e)\n",
    "  else:\n",
    "    try:\n",
    "      response = json.loads(json_response)\n",
    "      items = response.get('itemListElement', [])\n",
    "      result_obj['accessed'] = True\n",
    "      if items:\n",
    "        item = items[0]\n",
    "        item_result = item.get('result', [])\n",
    "        result_obj['description'] = item_result.get('description', '')\n",
    "        det_desc = item_result.get('detailedDescription', '')\n",
    "        if not result_obj['kg_id']:\n",
    "          result_obj['kg_id'] = item_result.get('@id', '').replace('kg:','')\n",
    "        if det_desc:\n",
    "          result_obj['description_extended'] = det_desc.get('articleBody','')\n",
    "          result_obj['description_license'] = det_desc.get('license','')\n",
    "          result_obj['description_url'] = det_desc.get('url','')\n",
    "        else:\n",
    "          result_obj['description_extended'] = ''\n",
    "          result_obj['description_license'] = ''\n",
    "          result_obj['description_url'] = ''\n",
    "        result_img = item_result.get('image', '')\n",
    "        if result_img:\n",
    "          result_obj['image_url'] = result_img.get('contentUrl', '')\n",
    "        result_obj['name'] = item_result.get('name', '')\n",
    "        result_obj['score'] = item.get('resultScore', 0.0)\n",
    "        result_obj['url'] = item_result.get('url', '')\n",
    "    except Exception as e:\n",
    "      result_obj['error'] = str(e)\n",
    "  return result_obj\n",
    "  \n",
    "def get_kg_from_name(obj):\n",
    "  if obj['accessed']:\n",
    "    return obj\n",
    "  params = {'query': obj['query']}\n",
    "  return _get_kg_meta(obj, params)\n",
    "  \n",
    "def get_kg_from_kg_id(obj):\n",
    "  if obj['accessed']:\n",
    "    return obj\n",
    "  params = {'ids': obj['kg_id']}\n",
    "  return _get_kg_meta(obj, params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'accessed': True,\n",
      " 'description': 'American singer',\n",
      " 'description_extended': 'Taylor Alison Swift is an American '\n",
      "                         \"singer-songwriter. As one of the world's leading \"\n",
      "                         'contemporary recording artists, she is known for '\n",
      "                         'narrative songs about her personal life, which has '\n",
      "                         'received widespread media coverage.\\n',\n",
      " 'description_license': 'https://en.wikipedia.org/wiki/Wikipedia:Text_of_Creative_Commons_Attribution-ShareAlike_3.0_Unported_License',\n",
      " 'description_url': 'https://en.wikipedia.org/wiki/Taylor_Swift',\n",
      " 'image_url': 'http://t0.gstatic.com/images?q=tbn:ANd9GcST848UJ0u31E6aoQfb2nnKZFyad7rwNF0ZLOCACGpu4jnboEzV',\n",
      " 'kg_id': '/m/0dl567',\n",
      " 'name': 'Taylor Swift',\n",
      " 'query': 'Taylor Swift',\n",
      " 'score': 1241.476318,\n",
      " 'url': 'http://taylorswift.com/'}\n"
     ]
    }
   ],
   "source": [
    "# test get from name\n",
    "obj = {'query': 'Taylor Swift', 'kg_id': '', 'score': 0.0, 'description': '', 'url':'', 'accessed': False}  # default\n",
    "result = get_kg_from_name(obj)\n",
    "pprint(obj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define thread mapping function\n",
    "def pool_map_persons(obj):\n",
    "  global pbar\n",
    "  pbar.update(1)\n",
    "  kg_obj = get_kg_from_name(obj)\n",
    "  return kg_obj"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# build mapped_person objects\n",
    "mapped_persons = []\n",
    "for fn in filenames:\n",
    "  obj = {'query': fn, 'kg_id': '', 'score': 0.0, 'description': '', 'url':'', 'accessed': False}\n",
    "  mapped_persons.append(obj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3107\n",
      "['aaron rodgers', 'aaron ruell', 'aaron staton', 'abel ferrara', 'abigail klein', 'abraham benrubi', 'abyshamble', 'adabel guerrero', 'adam ant', 'adam buxton']\n"
     ]
    }
   ],
   "source": [
    "print(len(mapped_persons))\n",
    "print(filenames[0:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4752a8e0280e4a58843a21401d9ed649",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3107), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1102/3107 remaining\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "882c60006b0d4a9e809297bbc1e86807",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3107), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "num_threads = 20\n",
    "pbar = tqdm(total=len(mapped_persons))\n",
    "\n",
    "num_non_accessed = sum(0 if x['accessed'] else 1 for x in mapped_persons)\n",
    "\n",
    "# convert to thread pool\n",
    "while num_non_accessed > 0:\n",
    "  print(f'{num_non_accessed}/{len(mapped_persons)} remaining')\n",
    "  pool = ThreadPool(num_threads)\n",
    "\n",
    "  # start threading\n",
    "  with tqdm(total=len(mapped_persons)) as pbar:\n",
    "    mapped_persons = pool.map(pool_map_persons, mapped_persons)\n",
    "\n",
    "  # close tqdm\n",
    "  pbar.close()\n",
    "\n",
    "  num_non_accessed = sum(0 if x['accessed'] else 1 for x in mapped_persons)\n",
    "  if num_non_accessed > 0:\n",
    "    print(f'{num_non_accessed} remaining. Sleeping...')\n",
    "    time.sleep(60*20)  # wait X minutes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'query': \"'Lee' George Quinones\", 'kg_id': '/m/08hvx1', 'score': 280.322754, 'description': 'Artist', 'url': 'http://www.leequinones.com/', 'accessed': True, 'description_extended': 'George Lee Quiñones is a Puerto Rican artist and actor. He is one of several artists to gain fame from the New York City Subway graffiti movement.\\n', 'description_license': 'https://en.wikipedia.org/wiki/Wikipedia:Text_of_Creative_Commons_Attribution-ShareAlike_3.0_Unported_License', 'description_url': 'https://en.wikipedia.org/wiki/Lee_Qui%C3%B1ones', 'name': 'Lee Quiñones'}\n"
     ]
    }
   ],
   "source": [
    "# test output for a person\n",
    "print(mapped_persons[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# reduce CC attribution string. the default strinf from Google Knowledge Graph is too verbose\n",
    "cc_long = 'https://en.wikipedia.org/wiki/Wikipedia:Text_of_Creative_Commons_Attribution-ShareAlike_3.0_Unported_License'\n",
    "cc_short = 'CC BY-SA 3.0'\n",
    "nchanged = 0\n",
    "for mapped_person in mapped_persons:\n",
    "  license = mapped_person.get('description_license', None)\n",
    "  if license == cc_long:\n",
    "    nchanged += 1\n",
    "    mapped_person['description_license'] = cc_short\n",
    "print(nchanged)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# find number not accessed\n",
    "n_empty = 0\n",
    "for mapped_person in mapped_persons:\n",
    "  if not mapped_person.get('accessed', False):\n",
    "    n_empty += 1\n",
    "    print(mapped_person['kg_id'])\n",
    "print(n_empty)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create dataframe for mapped persons\n",
    "df_mapped_persons = pd.DataFrame.from_dict(mapped_persons)\n",
    "df_mapped_persons.index.name = 'index'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check output\n",
    "df_mapped_persons.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save\n",
    "fp_out = '/data_store_hdd/datasets/people/imdb_wiki/metadata/identity_kg.csv'\n",
    "df_mapped_persons.to_csv(fp_out, encoding = 'utf-16')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create small version\n",
    "limit = 1000\n",
    "fpp_out = Path(fp_out)\n",
    "fp_out_sm = join(fpp_out.parent, f'{fpp_out.stem}_0_{limit}.csv')\n",
    "df_mapped_persons_sm = pd.DataFrame.from_dict(mapped_persons[0:limit])\n",
    "df_mapped_persons_sm.index.name = 'index'\n",
    "df_mapped_persons_sm.to_csv(fp_out_sm, encoding = 'utf-16')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for later, check similarity score to othyer identity kg CSVs\n",
    "from difflib import SequenceMatcher\n",
    "def similar(a, b):\n",
    "    return SequenceMatcher(None, a, b).ratio()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:megapixels]",
   "language": "python",
   "name": "conda-env-megapixels-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}