{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# IMDB-WIKI Knowledge Graph\n", "\n", "- convert names to Knowledge Graph entity IDs\n", "- The `imdb.mat` file contains only full names, need KG ids `/m/12345`" ] }, { "cell_type": "code", "execution_count": 87, "metadata": {}, "outputs": [], "source": [ "%reload_ext autoreload\n", "%autoreload 2\n", "\n", "import os\n", "import os.path as osp\n", "from os.path import join\n", "from glob import glob\n", "from pathlib import Path\n", "import random\n", "import math\n", "from datetime import datetime\n", "import requests\n", "import json\n", "import time\n", "from pprint import pprint\n", "from multiprocessing.pool import ThreadPool\n", "import threading\n", "import urllib.request\n", "\n", "from tqdm import tqdm_notebook as tqdm\n", "import pandas as pd\n", "from scipy.io import loadmat\n", "import numpy as np\n", "%matplotlib inline\n", "import matplotlib.pyplot as plt" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Load IMDB Metadata" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
celeb_iddobfilepathgendernamex1x2y1y2year_photo
index
064881900-5-1101/nm0000001_rm124825600_1899-5-10_1968.jpgmFred Astaire1072.9261214.784161.838303.6961968
164881900-5-1101/nm0000001_rm3343756032_1899-5-10_1970.jpgmFred Astaire477.184622.592100.352245.7601970
\n", "
" ], "text/plain": [ " celeb_id dob filepath \\\n", "index \n", "0 6488 1900-5-11 01/nm0000001_rm124825600_1899-5-10_1968.jpg \n", "1 6488 1900-5-11 01/nm0000001_rm3343756032_1899-5-10_1970.jpg \n", "\n", " gender name x1 x2 y1 y2 year_photo \n", "index \n", "0 m Fred Astaire 1072.926 1214.784 161.838 303.696 1968 \n", "1 m Fred Astaire 477.184 622.592 100.352 245.760 1970 " ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "fp_meta_imdb = '/data_store_hdd/datasets/people/imdb_wiki/metadata/imdb_mat.csv'\n", "df_meta_imdb = pd.read_csv(fp_meta_imdb).set_index('index')\n", "df_meta_imdb.head(2)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Google Knowledge Graph API" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [], "source": [ "# read API key\n", "api_key = open('/work/megapixels_dev/3rdparty/knowledge-graph-api/.api_key').read()\n", "url_kg_api = 'https://kgsearch.googleapis.com/v1/entities:search'" ] }, { "cell_type": "code", "execution_count": 45, "metadata": {}, "outputs": [], "source": [ "def _get_kg_meta(result_obj, params):\n", " global api_key, url_kg_api\n", " \n", " params['indent'] = True\n", " params['key'] = api_key\n", " params['limit'] = 1\n", " \n", " url = f'{url_kg_api}?{urllib.parse.urlencode(params)}'\n", " try:\n", " json_response = urllib.request.urlopen(url).read()\n", " except Exception as e:\n", " result['error'] = str(e)\n", " else:\n", " try:\n", " response = json.loads(json_response)\n", " items = response.get('itemListElement', [])\n", " result_obj['accessed'] = True\n", " if items:\n", " item = items[0]\n", " item_result = item.get('result', [])\n", " result_obj['description'] = item_result.get('description', '')\n", " det_desc = item_result.get('detailedDescription', '')\n", " if not result_obj['kg_id']:\n", " result_obj['kg_id'] = item_result.get('@id', '').replace('kg:','')\n", " if det_desc:\n", " result_obj['description_extended'] = det_desc.get('articleBody','')\n", " result_obj['description_license'] = det_desc.get('license','')\n", " result_obj['description_url'] = det_desc.get('url','')\n", " else:\n", " result_obj['description_extended'] = ''\n", " result_obj['description_license'] = ''\n", " result_obj['description_url'] = ''\n", " result_img = item_result.get('image', '')\n", " if result_img:\n", " result_obj['image_url'] = result_img.get('contentUrl', '')\n", " result_obj['name'] = item_result.get('name', '')\n", " result_obj['score'] = item.get('resultScore', 0.0)\n", " result_obj['url'] = item_result.get('url', '')\n", " except Exception as e:\n", " result_obj['error'] = str(e)\n", " return result_obj\n", " \n", "def get_kg_from_name(obj):\n", " if obj['accessed']:\n", " return obj\n", " params = {'query': obj['query']}\n", " return _get_kg_meta(obj, params)\n", " \n", "def get_kg_from_kg_id(obj):\n", " if obj['accessed']:\n", " return obj\n", " params = {'ids': obj['kg_id']}\n", " return _get_kg_meta(obj, params)" ] }, { "cell_type": "code", "execution_count": 46, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'accessed': True,\n", " 'description': 'American singer',\n", " 'description_extended': 'Taylor Alison Swift is an American '\n", " \"singer-songwriter. As one of the world's leading \"\n", " 'contemporary recording artists, she is known for '\n", " 'narrative songs about her personal life, which has '\n", " 'received widespread media coverage.\\n',\n", " 'description_license': 'https://en.wikipedia.org/wiki/Wikipedia:Text_of_Creative_Commons_Attribution-ShareAlike_3.0_Unported_License',\n", " 'description_url': 'https://en.wikipedia.org/wiki/Taylor_Swift',\n", " 'image_url': 'http://t0.gstatic.com/images?q=tbn:ANd9GcST848UJ0u31E6aoQfb2nnKZFyad7rwNF0ZLOCACGpu4jnboEzV',\n", " 'kg_id': '/m/0dl567',\n", " 'name': 'Taylor Swift',\n", " 'query': 'Taylor Swift',\n", " 'score': 1241.476318,\n", " 'url': 'http://taylorswift.com/'}\n" ] } ], "source": [ "# make a test query to check if API works\n", "obj = {'query': 'Taylor Swift', 'kg_id': '', 'score': 0.0, 'description': '', 'url':'', 'accessed': False} # default\n", "result = get_kg_from_name(obj)\n", "pprint(obj)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "obj = {'query': 'Taylor Swift', 'score': 0.0, 'description': '', 'url':'', 'accessed': False} # default\n", "result = get_kg_from_id(obj)" ] }, { "cell_type": "code", "execution_count": 49, "metadata": {}, "outputs": [], "source": [ "# build mapped_person objects\n", "mapped_persons = []\n", "count = 0\n", "df_person_groups = df_meta_imdb.groupby('name')\n", "for group_name, df_name_group in df_person_groups:\n", " obj = {'query': group_name, 'kg_id': '', 'score': 0.0, 'description': '', 'url':'', 'accessed': False}\n", " mapped_persons.append(obj)" ] }, { "cell_type": "code", "execution_count": 55, "metadata": {}, "outputs": [], "source": [ "# define thread mapping function\n", "def pool_map_persons(obj):\n", " global pbar\n", " pbar.update(1)\n", " kg_obj = get_kg_from_name(obj)\n", " return kg_obj" ] }, { "cell_type": "code", "execution_count": 79, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "87f6a2be42284199b8a67458f4090497", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(IntProgress(value=0, max=20284), HTML(value='')))" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "0/20284 remaining\n" ] } ], "source": [ "num_threads = 2\n", "pbar = tqdm(total=len(mapped_persons))\n", "\n", "num_non_accessed = sum(0 if x['accessed'] else 1 for x in mapped_persons)\n", "print(f'{num_non_accessed}/{len(mapped_persons)} remaining')\n", "\n", "# convert to thread pool\n", "while num_non_accessed > 0:\n", " print(f'{num_non_accessed}/{len(mapped_persons)} remaining')\n", " pool = ThreadPool(num_threads)\n", "\n", " # start threading\n", " with tqdm(total=len(mapped_persons)) as pbar:\n", " mapped_persons = pool.map(pool_map_persons, mapped_persons)\n", "\n", " # close tqdm\n", " pbar.close()\n", "\n", " num_non_accessed = sum(0 if x['accessed'] else 1 for x in mapped_persons)\n", " if num_non_accessed > 0:\n", " print(f'{num_non_accessed}/{len(mapped_persons)} remaining. Sleeping...')\n", " time.sleep(60*20) # wait X minutes" ] }, { "cell_type": "code", "execution_count": 80, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'query': \"'Lee' George Quinones\", 'kg_id': '/m/08hvx1', 'score': 280.322754, 'description': 'Artist', 'url': 'http://www.leequinones.com/', 'accessed': True, 'description_extended': 'George Lee Quiñones is a Puerto Rican artist and actor. He is one of several artists to gain fame from the New York City Subway graffiti movement.\\n', 'description_license': 'https://en.wikipedia.org/wiki/Wikipedia:Text_of_Creative_Commons_Attribution-ShareAlike_3.0_Unported_License', 'description_url': 'https://en.wikipedia.org/wiki/Lee_Qui%C3%B1ones', 'name': 'Lee Quiñones'}\n" ] } ], "source": [ "# test output for a person\n", "print(mapped_persons[0])" ] }, { "cell_type": "code", "execution_count": 82, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0\n" ] } ], "source": [ "# reduce CC attribution string. the default strinf from Google Knowledge Graph is too verbose\n", "cc_long = 'https://en.wikipedia.org/wiki/Wikipedia:Text_of_Creative_Commons_Attribution-ShareAlike_3.0_Unported_License'\n", "cc_short = 'CC BY-SA 3.0'\n", "nchanged = 0\n", "for mapped_person in mapped_persons:\n", " license = mapped_person.get('description_license', None)\n", " if license == cc_long:\n", " nchanged += 1\n", " mapped_person['description_license'] = cc_short\n", "print(nchanged)" ] }, { "cell_type": "code", "execution_count": 83, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0\n" ] } ], "source": [ "# find number not accessed\n", "n_empty = 0\n", "for mapped_person in mapped_persons:\n", " if not mapped_person.get('accessed', False):\n", " n_empty += 1\n", " print(mapped_person['kg_id'])\n", "print(n_empty)" ] }, { "cell_type": "code", "execution_count": 84, "metadata": {}, "outputs": [], "source": [ "# create dataframe for mapped persons\n", "df_mapped_persons = pd.DataFrame.from_dict(mapped_persons)\n", "df_mapped_persons.index.name = 'index'" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# check output\n", "df_mapped_persons.head()" ] }, { "cell_type": "code", "execution_count": 85, "metadata": {}, "outputs": [], "source": [ "# save\n", "fp_out = '/data_store_hdd/datasets/people/imdb_wiki/metadata/identity_kg.csv'\n", "df_mapped_persons.to_csv(fp_out, encoding = 'utf-16')" ] }, { "cell_type": "code", "execution_count": 88, "metadata": {}, "outputs": [], "source": [ "# create small version\n", "limit = 1000\n", "fpp_out = Path(fp_out)\n", "fp_out_sm = join(fpp_out.parent, f'{fpp_out.stem}_0_{limit}.csv')\n", "df_mapped_persons_sm = pd.DataFrame.from_dict(mapped_persons[0:limit])\n", "df_mapped_persons_sm.index.name = 'index'\n", "df_mapped_persons_sm.to_csv(fp_out_sm, encoding = 'utf-16')" ] } ], "metadata": { "kernelspec": { "display_name": "Python [conda env:megapixels]", "language": "python", "name": "conda-env-megapixels-py" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.6" } }, "nbformat": 4, "nbformat_minor": 2 }