add utils for analyzing identities

1 files changed, 792 insertions, 0 deletions

diff --git a/megapixels/notebooks/datasets/knowledge_graph/identity.ipynb b/megapixels/notebooks/datasets/knowledge_graph/identity.ipynb
new file mode 100644
index 00000000..81a74faf
--- /dev/null
+++ b/megapixels/notebooks/datasets/knowledge_graph/identity.ipynb
@@ -0,0 +1,792 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Knowledge Graph Identities\n",
+    "\n",
+    "- convert filename-names to names\n",
+    "- fetch Google Knowledge Graph entity IDs for each name\n",
+    "- save KG IDs to CSV"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 48,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%reload_ext autoreload\n",
+    "%autoreload 2\n",
+    "\n",
+    "import os\n",
+    "import os.path as osp\n",
+    "from os.path import join\n",
+    "from glob import glob\n",
+    "from pathlib import Path\n",
+    "import random\n",
+    "import math\n",
+    "from datetime import datetime\n",
+    "import requests\n",
+    "import json\n",
+    "import time\n",
+    "from pprint import pprint\n",
+    "from multiprocessing.pool import ThreadPool\n",
+    "import threading\n",
+    "import urllib.request\n",
+    "import difflib\n",
+    "import unidecode\n",
+    "import slugify\n",
+    "\n",
+    "from tqdm import tqdm_notebook as tqdm\n",
+    "import pandas as pd\n",
+    "from scipy.io import loadmat\n",
+    "import numpy as np\n",
+    "%matplotlib inline\n",
+    "import matplotlib.pyplot as plt\n",
+    "\n",
+    "import sys\n",
+    "sys.path.append('/work/megapixels_dev/megapixels')\n",
+    "from app.utils import api_utils\n",
+    "from app.settings import types"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Get List of Names"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_names(enum_dataset):\n",
+    "  if enum_dataset == types.Dataset.LFW:\n",
+    "    dir_lfw = '/data_store_hdd/datasets/people/lfw/media/original/'\n",
+    "    names = [x.replace('_', ' ') for x in os.listdir(dir_lfw)]\n",
+    "  elif enum_dataset == types.Dataset.YOUTUBE_FACES:\n",
+    "    names = [x for x in names if 'labeled faces.txt' not in x]\n",
+    "  return names"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "['Kim Clijsters', 'William Rosenberg', 'John Brady', 'Juan Ignacio Chela', 'Floyd Keith', 'Sam Gerald', 'Imad Khadduri', 'Anna Kournikova', 'Jacques Rogge', 'Wilbert Elki Meza Majino']\n"
+     ]
+    }
+   ],
+   "source": [
+    "names = get_names(types.Dataset.LFW)\n",
+    "print(names[0:10])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Google Knowledge Graph API"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# read API key\n",
+    "api_key = open('/work/megapixels_dev/env/google_knowledge_graph_api.env').read()\n",
+    "kg_api = api_utils.GoogleKnowledgeGraph(api_key)\n",
+    "wp_api = api_utils.WikipediaAPI()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 241,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "wp\n",
+      "{'wp_accessed': True, 'wp_description': '', 'wp_name': '', 'wp_page_id': ''}\n",
+      "kg\n",
+      "{'kg_accessed': True,\n",
+      " 'kg_bio': '',\n",
+      " 'kg_bio_url': '',\n",
+      " 'kg_description': '',\n",
+      " 'kg_id': '',\n",
+      " 'kg_image_url': '',\n",
+      " 'kg_name': '',\n",
+      " 'kg_score': 0,\n",
+      " 'kg_url': '',\n",
+      " 'query': 'Jeff Dederian'}\n"
+     ]
+    }
+   ],
+   "source": [
+    "#wp_api.test_access()\n",
+    "print('wp')\n",
+    "pprint(wp_api.get_meta({'query': 'Florecita Cobian'}))\n",
+    "print('kg')\n",
+    "pprint(kg_api.get_kg_from_name({'query':'Jeff Dederian'}))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Test Name Similarity Matching"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 242,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def same_person(query, name, sim_min=.9, word_match_min=0.75, verbose=False):\n",
+    "  if name == '':\n",
+    "    return False\n",
+    "  # check and remove if WP added parenthesis\n",
+    "  if '(' in name and ')' in name:\n",
+    "    name = name.split('(')[0]\n",
+    "  \n",
+    "  # then strip spaces and split into list\n",
+    "  query_strings = [unidecode.unidecode(x.strip().lower()) for x in query.strip().split(' ')]  # query\n",
+    "  result_strings = [unidecode.unidecode(x.strip().lower()) for x in name.strip().split(' ')]  # result\n",
+    "  min_str_len = min(len(result_strings), len(query_strings))\n",
+    "  # match each word in the query\n",
+    "  matched_strings = []\n",
+    "  \n",
+    "  for i in range(len(query_strings)):\n",
+    "    # for each word in the shorter text string\n",
+    "    result_strings_tmp = result_strings.copy()\n",
+    "    for j in range(len(result_strings_tmp)):\n",
+    "      a = query_strings[i]\n",
+    "      b = result_strings_tmp[j]\n",
+    "      # make a the shorter string\n",
+    "      lengths = [len(a), len(b)]\n",
+    "      min_ratio = (min(lengths) / max(lengths) * .75)\n",
+    "      ratio = difflib.SequenceMatcher(a=a.lower(), b=b.lower()).ratio()\n",
+    "      result = (ratio >= min_ratio)\n",
+    "      if verbose:\n",
+    "        print(f'comapre \"{a}\" to \"{b}\" ratio was: {ratio:.2f} min: {min_ratio:.2}, passed: {result}')\n",
+    "      if result:\n",
+    "        # remove this item from result strings\n",
+    "        matched_string = result_strings.pop(j)\n",
+    "        matched_strings.append(matched_string)\n",
+    "        break  # exit loop and use shortened result string haystack\n",
+    "\n",
+    "  matched = len(matched_strings) >= min_str_len\n",
+    "  if verbose:\n",
+    "      print(f'{matched} because {len(matched_strings)} >= {min_str_len}')\n",
+    "  return matched"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 245,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(Adoor Gopalakrishnan == Adoors Gopalakarishnan ok) = True\n",
+      "\n",
+      "comapre \"dave\" to \"david\" ratio was: 0.67 min: 0.6, passed: True\n",
+      "comapre \"letterman\" to \"letterman\" ratio was: 1.00 min: 0.75, passed: True\n",
+      "True because 2 >= 2\n",
+      "(David Letterman == Dave Letterman) = True\n",
+      "\n",
+      "comapre \"charles\" to \"charles\" ratio was: 1.00 min: 0.75, passed: True\n",
+      "comapre \"dickens\" to \"booker\" ratio was: 0.31 min: 0.64, passed: False\n",
+      "False because 1 >= 2\n",
+      "(Charles Booker == Charles Dickens) = False\n",
+      "\n",
+      "comapre \"donald\" to \"don\" ratio was: 0.67 min: 0.38, passed: True\n",
+      "comapre \"trump\" to \"j.\" ratio was: 0.00 min: 0.3, passed: False\n",
+      "comapre \"trump\" to \"trump\" ratio was: 1.00 min: 0.75, passed: True\n",
+      "True because 2 >= 2\n",
+      "(Don J. Trump == Donald Trump) = True\n",
+      "\n",
+      "comapre \"wang\" to \"wang\" ratio was: 1.00 min: 0.75, passed: True\n",
+      "comapre \"fei\" to \"fei\" ratio was: 1.00 min: 0.75, passed: True\n",
+      "True because 2 >= 2\n",
+      "(Wang Fei (female footballer) == Wang Fei) = True\n"
+     ]
+    }
+   ],
+   "source": [
+    "test_sim_match = True\n",
+    "if test_sim_match:\n",
+    "  # Test name similarity search\n",
+    "  query = 'Adoors Gopalakarishnan ok'\n",
+    "  wp_name = 'Adoor Gopalakrishnan'\n",
+    "  matched = same_person(query, wp_name)\n",
+    "  print(f'({wp_name} == {query}) = {matched}')\n",
+    "  print('')\n",
+    "\n",
+    "  query = 'Dave Letterman'\n",
+    "  wp_name = 'David Letterman'\n",
+    "  matched = same_person(query, wp_name, verbose=True)\n",
+    "  print(f'({wp_name} == {query}) = {matched}')\n",
+    "  print('')\n",
+    "\n",
+    "  query = 'Charles Dickens'\n",
+    "  wp_name = 'Charles Booker'\n",
+    "  matched = same_person(query, wp_name, verbose=True)\n",
+    "  print(f'({wp_name} == {query}) = {matched}')\n",
+    "  print('')\n",
+    "\n",
+    "  query = 'Donald Trump'\n",
+    "  wp_name = 'Don J. Trump'\n",
+    "  matched = same_person(query, wp_name, verbose=True)\n",
+    "  print(f'({wp_name} == {query}) = {matched}')\n",
+    "  print('')\n",
+    "  \n",
+    "  query = 'Wang Fei'\n",
+    "  kg_name = 'Faye Wong'\n",
+    "  wp_name = 'Wang Fei (female footballer)'\n",
+    "  matched = same_person(query, wp_name, verbose=True)\n",
+    "  print(f'({wp_name} == {query}) = {matched}')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 246,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# define thread mapping function\n",
+    "def pool_map_persons(obj):\n",
+    "  global pbar\n",
+    "  pbar.update(1)\n",
+    "  kg_obj = kg_api.get_kg_from_name(obj)\n",
+    "  wp_obj = wp_api.get_meta(obj)\n",
+    "  person_obj = {**kg_obj, **wp_obj}\n",
+    "  return person_obj\n",
+    "\n",
+    "def num_non_accessed(mps):\n",
+    "  return sum(0 if (x.get('kg_accessed', False) and x.get('wp_accessed', False)) else 1 for x in mps)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Load existing CSV"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# load existing CSV\n",
+    "fp_csv = '/data_store_hdd/datasets/people/lfw/metadata/identity_kg.csv'\n",
+    "df = pd.read_csv(fp_csv, encoding = 'utf-16').set_index('index')\n",
+    "# fill nulls\n",
+    "df.fillna('', inplace = True)\n",
+    "mapped_persons = df.to_dict('records')\n",
+    "# add columns\n",
+    "for mp in mapped_persons:\n",
+    "  mp['wp_error'] = ''\n",
+    "  mp['kg_error'] = ''"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Get Knowledge Graph Data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "5507f5c19de746df94aa5445e3c7cf46",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, max=5749), HTML(value='')))"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "832/5749 remaining\n",
+      "832/5749 remaining. Using 5 threads\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "411d08f873174d13a1de1f8b21f9f993",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, max=5749), HTML(value='')))"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Done. 0 remaining.\n"
+     ]
+    }
+   ],
+   "source": [
+    "num_threads_max = 5\n",
+    "sleep_min = 1\n",
+    "pbar = tqdm(total=len(mapped_persons))\n",
+    "\n",
+    "nna = num_non_accessed(mapped_persons)\n",
+    "print(f'{nna}/{len(mapped_persons)} remaining')\n",
+    "\n",
+    "# convert to thread pool\n",
+    "while nna > 0:\n",
+    "  num_threads = max(1, min(num_threads_max, nna))\n",
+    "  print(f'{nna}/{len(mapped_persons)} remaining. Using {num_threads} threads')\n",
+    "  pool = ThreadPool(num_threads)\n",
+    "\n",
+    "  # start threading\n",
+    "  with tqdm(total=len(mapped_persons)) as pbar:\n",
+    "    mapped_persons = pool.map(pool_map_persons, mapped_persons)\n",
+    "\n",
+    "  # close tqdm\n",
+    "  pbar.close()\n",
+    "\n",
+    "  nna = num_non_accessed(mapped_persons)\n",
+    "  if nna > 0:\n",
+    "    print(f'{nna} remaining. Sleeping for {sleep_min} minutes...')\n",
+    "    time.sleep(60 * sleep_min)\n",
+    "\n",
+    "print(f'Done. {nna} remaining.')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Get Wikipedia API data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 220,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [],
+   "source": [
+    "for i, mp in enumerate(mapped_persons):\n",
+    "  kg_name = mp.get('kg_name')\n",
+    "  wp_name = mp.get('wp_name')\n",
+    "  query = mp.get('query')\n",
+    "  name_orig = mp.get('source_name')\n",
+    "  kg_score = int(mp.get('kg_score',0))\n",
+    "\n",
+    "  kg_matches = same_person(name_orig, kg_name)\n",
+    "  wp_matches = same_person(name_orig, wp_name)\n",
+    "\n",
+    "  if kg_matches and wp_matches and kg_score > 100:\n",
+    "    # very likely a match, confirm it\n",
+    "    match_status = 2  # supermatch\n",
+    "    # default to using wp because descriptions are more appropriate/udpated\n",
+    "    source = 'wp'\n",
+    "  elif kg_matches and wp_matches:\n",
+    "    match_status = 1\n",
+    "    # default to using wp because descriptions are more appropriate/udpated\n",
+    "    source = 'wp'\n",
+    "  elif kg_matches and not wp_matches:\n",
+    "    # if the KG score is medium-high, but wp failed, needs review\n",
+    "    source = 'kg'\n",
+    "    match_status = 0\n",
+    "  elif wp_matches and not kg_matches:\n",
+    "    # if wikipedia text matched the query, then confirm\n",
+    "    source = 'wp'\n",
+    "    match_status = 0\n",
+    "  else:\n",
+    "    # no information available\n",
+    "    match_status = -1\n",
+    "    source = None\n",
+    "      \n",
+    "  slug = slugify.slugify(name_orig, separator='_')\n",
+    "  mp_bio = mp.get('kg_bio', '')\n",
+    "  wp_desc = mp.get('wp_description', '')\n",
+    "  source_url = f\"http://vis-www.cs.umass.edu/lfw/person/{name_orig.replace(' ', '_')}.html\"\n",
+    "  \n",
+    "  if source == 'kg':\n",
+    "    # google knowledge graph\n",
+    "    mp_name = mp['kg_name']\n",
+    "    mp_description = mp.get('kg_description', '')\n",
+    "  elif source == 'wp':\n",
+    "    # wikipedia\n",
+    "    mp_name = mp['wp_name']\n",
+    "    mp_description = mp.get('wp_description', '')\n",
+    "  \n",
+    "  if 'disambiguation' in wp_desc.lower():\n",
+    "    #print(f\"disambiguate: {name_orig}\")\n",
+    "    match_status = 0  # needs review if \"disambiguation appears\"\n",
+    "    mp_name = ''\n",
+    "    mp_description = ''\n",
+    "    mp_bio = ''\n",
+    "  \n",
+    "  mp['source_url'] = source_url\n",
+    "  mp['mp_slug'] = slug\n",
+    "  mp['matched'] = match_status\n",
+    "  mp['mp_bio'] = mp_bio\n",
+    "  mp['mp_name'] = mp_name\n",
+    "  mp['mp_description'] = mp_description"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 221,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "match: 4359\n",
+      "review: 718\n",
+      "fail: 672\n",
+      "no kg accessed: 0\n",
+      "no wp accessed: 0\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(f\"match: {sum(1 if (x.get('matched') > 0) else 0 for x in mapped_persons)}\")\n",
+    "print(f\"review: {sum(1 if (x.get('matched') == 0) else 0 for x in mapped_persons)}\")\n",
+    "print(f\"fail: {sum(1 if (x.get('matched') == -1) else 0 for x in mapped_persons)}\")\n",
+    "\n",
+    "print(f\"no kg accessed: {sum(0 if (x.get('kg_accessed', False)) else 1 for x in mapped_persons)}\")\n",
+    "print(f\"no wp accessed: {sum(0 if (x.get('wp_accessed', False)) else 1 for x in mapped_persons)}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Save data to CSV"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 235,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# create dataframe for mapped persons\n",
+    "df_mapped_persons = pd.DataFrame.from_dict(mapped_persons)\n",
+    "df_mapped_persons.index.name = 'index'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 236,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# save\n",
+    "fp_out = f'/data_store_hdd/datasets/people/lfw/metadata/identity_kg.csv'\n",
+    "df_mapped_persons.drop(['kg_accessed', 'wp_accessed', 'kg_error', 'wp_error'], axis=1, inplace=True)\n",
+    "df_mapped_persons.to_csv(fp_out, encoding = 'utf-16')\n",
+    "# create small version\n",
+    "limit = 1000\n",
+    "fpp_out = Path(fp_out)\n",
+    "fp_out_sm = join(fpp_out.parent, f'{fpp_out.stem}_0_{limit}.csv')\n",
+    "df_mapped_persons_sm = pd.DataFrame.from_dict(mapped_persons[0:limit])\n",
+    "df_mapped_persons_sm.index.name = 'index'\n",
+    "df_mapped_persons_sm.to_csv(fp_out_sm, encoding = 'utf-16')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 237,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>kg_bio</th>\n",
+       "      <th>kg_bio_url</th>\n",
+       "      <th>kg_description</th>\n",
+       "      <th>kg_id</th>\n",
+       "      <th>kg_image_url</th>\n",
+       "      <th>kg_name</th>\n",
+       "      <th>kg_score</th>\n",
+       "      <th>kg_url</th>\n",
+       "      <th>matched</th>\n",
+       "      <th>mp_bio</th>\n",
+       "      <th>mp_description</th>\n",
+       "      <th>mp_name</th>\n",
+       "      <th>mp_slug</th>\n",
+       "      <th>query</th>\n",
+       "      <th>source</th>\n",
+       "      <th>source_name</th>\n",
+       "      <th>source_url</th>\n",
+       "      <th>wp_description</th>\n",
+       "      <th>wp_name</th>\n",
+       "      <th>wp_page_id</th>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>index</th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Kim Antonie Lode Clijsters is a Belgian former...</td>\n",
+       "      <td>https://en.wikipedia.org/wiki/Kim_Clijsters</td>\n",
+       "      <td>Belgian tennis player</td>\n",
+       "      <td>/m/01m_gh</td>\n",
+       "      <td>http://t3.gstatic.com/images?q=tbn:ANd9GcQ4yRK...</td>\n",
+       "      <td>Kim Clijsters</td>\n",
+       "      <td>618.272705</td>\n",
+       "      <td></td>\n",
+       "      <td>2</td>\n",
+       "      <td>Kim Antonie Lode Clijsters is a Belgian former...</td>\n",
+       "      <td>Belgian tennis player</td>\n",
+       "      <td>Kim Clijsters</td>\n",
+       "      <td>kim_clijsters</td>\n",
+       "      <td>Kim Clijsters</td>\n",
+       "      <td>lfw</td>\n",
+       "      <td>Kim_Clijsters</td>\n",
+       "      <td>http://vis-www.cs.umass.edu/lfw/person/Kim_Cli...</td>\n",
+       "      <td>Belgian tennis player</td>\n",
+       "      <td>Kim Clijsters</td>\n",
+       "      <td>262793</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>William Rosenberg was an American entrepreneur...</td>\n",
+       "      <td>https://en.wikipedia.org/wiki/William_Rosenberg</td>\n",
+       "      <td>American entrepreneur</td>\n",
+       "      <td>/m/07dy4z</td>\n",
+       "      <td></td>\n",
+       "      <td>William Rosenberg</td>\n",
+       "      <td>367.879730</td>\n",
+       "      <td></td>\n",
+       "      <td>2</td>\n",
+       "      <td>William Rosenberg was an American entrepreneur...</td>\n",
+       "      <td>American businessman</td>\n",
+       "      <td>William Rosenberg</td>\n",
+       "      <td>william_rosenberg</td>\n",
+       "      <td>William Rosenberg</td>\n",
+       "      <td>lfw</td>\n",
+       "      <td>William_Rosenberg</td>\n",
+       "      <td>http://vis-www.cs.umass.edu/lfw/person/William...</td>\n",
+       "      <td>American businessman</td>\n",
+       "      <td>William Rosenberg</td>\n",
+       "      <td>2.44981e+06</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                  kg_bio  \\\n",
+       "index                                                      \n",
+       "0      Kim Antonie Lode Clijsters is a Belgian former...   \n",
+       "1      William Rosenberg was an American entrepreneur...   \n",
+       "\n",
+       "                                            kg_bio_url         kg_description  \\\n",
+       "index                                                                           \n",
+       "0          https://en.wikipedia.org/wiki/Kim_Clijsters  Belgian tennis player   \n",
+       "1      https://en.wikipedia.org/wiki/William_Rosenberg  American entrepreneur   \n",
+       "\n",
+       "           kg_id                                       kg_image_url  \\\n",
+       "index                                                                 \n",
+       "0      /m/01m_gh  http://t3.gstatic.com/images?q=tbn:ANd9GcQ4yRK...   \n",
+       "1      /m/07dy4z                                                      \n",
+       "\n",
+       "                 kg_name    kg_score kg_url  matched  \\\n",
+       "index                                                  \n",
+       "0          Kim Clijsters  618.272705               2   \n",
+       "1      William Rosenberg  367.879730               2   \n",
+       "\n",
+       "                                                  mp_bio  \\\n",
+       "index                                                      \n",
+       "0      Kim Antonie Lode Clijsters is a Belgian former...   \n",
+       "1      William Rosenberg was an American entrepreneur...   \n",
+       "\n",
+       "              mp_description            mp_name            mp_slug  \\\n",
+       "index                                                                \n",
+       "0      Belgian tennis player      Kim Clijsters      kim_clijsters   \n",
+       "1       American businessman  William Rosenberg  william_rosenberg   \n",
+       "\n",
+       "                   query source        source_name  \\\n",
+       "index                                                \n",
+       "0          Kim Clijsters    lfw      Kim_Clijsters   \n",
+       "1      William Rosenberg    lfw  William_Rosenberg   \n",
+       "\n",
+       "                                              source_url  \\\n",
+       "index                                                      \n",
+       "0      http://vis-www.cs.umass.edu/lfw/person/Kim_Cli...   \n",
+       "1      http://vis-www.cs.umass.edu/lfw/person/William...   \n",
+       "\n",
+       "              wp_description            wp_name   wp_page_id  \n",
+       "index                                                         \n",
+       "0      Belgian tennis player      Kim Clijsters       262793  \n",
+       "1       American businessman  William Rosenberg  2.44981e+06  "
+      ]
+     },
+     "execution_count": 237,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_mapped_persons.head(2)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Clean data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 225,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [],
+   "source": [
+    "for mp in mapped_persons:\n",
+    "  mp['source_name'] = mp['source_name'].replace(' ', '_')\n",
+    "#   mp['kg_description'] = mp['kg_description'].strip()\n",
+    "#   mp['kg_name'] = mp['kg_name'].strip()\n",
+    "#   mp['kg_bio_url'] = mp['kg_bio_url'].strip()\n",
+    "#   mp['kg_bio'] = mp['kg_bio'].strip()\n",
+    "#   mp['kg_url'] = mp['kg_url'].strip()\n",
+    "  \n",
+    "#   mp['wp_description'] = mp['wp_description'].strip()\n",
+    "#   mp['wp_name'] = mp['wp_name'].strip()\n",
+    "  \n",
+    "#   mp['mp_name'] = ''\n",
+    "#   mp['mp_bio'] = ''\n",
+    "#   mp['mp_description'] = ''\n",
+    "#   mp['mp_slug'] = ''\n",
+    "  \n",
+    "  #mp.setdefault('kg_description','')\n",
+    "#   if mp.get('kg_score', 0) == 0:\n",
+    "#     mp['kg_image_url'] = ''\n",
+    "#     mp['kg_bio_url'] = ''\n",
+    "#     mp['kg_id'] = ''\n",
+    "#     mp['kg_url'] = ''\n",
+    "#     mp['kg_description'] = ''\n",
+    "#     mp['kg_bio_url'] = ''\n",
+    "#     mp['kg_name'] = ''\n",
+    "#   if mp['kg_url'] == [] or mp['kg_url'] == '[]':\n",
+    "#     mp['kg_url'] = ''\n",
+    "\n",
+    "  try:\n",
+    "    _ = mp.pop('wp_bio')\n",
+    "  except:\n",
+    "    pass"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python [conda env:megapixels]",
+   "language": "python",
+   "name": "conda-env-megapixels-py"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}