new nbs

1 files changed, 800 insertions, 0 deletions

diff --git a/megapixels/notebooks/datasets/lfw/split_records.ipynb b/megapixels/notebooks/datasets/lfw/split_records.ipynb
new file mode 100644
index 00000000..b7f1c8d6
--- /dev/null
+++ b/megapixels/notebooks/datasets/lfw/split_records.ipynb
@@ -0,0 +1,800 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Split CSVs into multiple files"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%reload_ext autoreload\n",
+    "%autoreload 2\n",
+    "\n",
+    "import os\n",
+    "from os.path import join\n",
+    "import math\n",
+    "from glob import glob\n",
+    "from random import randint\n",
+    "from pathlib import Path\n",
+    "\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "%matplotlib inline\n",
+    "import matplotlib.pyplot as plt\n",
+    "from tqdm import tqdm\n",
+    "\n",
+    "import sys\n",
+    "sys.path.append('/work/megapixels_dev/megapixels/')\n",
+    "from app.utils import file_utils"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "DATA_STORE_NAS = '/data_store_nas/'\n",
+    "DATA_STORE_SSD = '/data_store_ssd/'\n",
+    "dir_dataset_ssd = 'apps/megapixels/datasets/lfw/'\n",
+    "dir_dataset_nas = 'datasets/people/lfw'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 63,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fp_records = join(DATA_STORE_NAS, dir_dataset_nas, 'records.csv')\n",
+    "fp_index = join(DATA_STORE_NAS, dir_dataset_nas, 'index.csv')\n",
+    "fp_sha256s = join(DATA_STORE_NAS, dir_dataset_nas, 'sha256s.csv')\n",
+    "fp_uuids = join(DATA_STORE_NAS, dir_dataset_nas, 'uuids.csv')\n",
+    "fp_rois = join(DATA_STORE_NAS, dir_dataset_nas, 'rois.csv')\n",
+    "fp_names_gender_kg = join(DATA_STORE_NAS, dir_dataset_nas, 'lfw_names_gender_kg.csv')\n",
+    "fp_identity_meta = join(DATA_STORE_NAS, dir_dataset_nas, 'identity_meta.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 77,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import difflib\n",
+    "def similarity(a, b):\n",
+    "  seq = difflib.SequenceMatcher(a=a.lower(), b=b.lower())\n",
+    "  return seq.ratio()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 48,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_records = pd.read_csv(fp_records).set_index('index')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 50,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Create index.csv, `index, sha256`\n",
+    "# drop old columns\n",
+    "df_records.drop(['ext', 'fn', 'subdir', 'uuid'], axis=1).to_csv(fp_index)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 51,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Create uuids.csv, `index, uuid`\n",
+    "# drop old columns\n",
+    "df_records.drop(['ext', 'fn', 'subdir', 'sha256'], axis=1).to_csv(fp_uuids)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Create identity_meta.csv, `index, name, description, gender`\n",
+    "# drop old columns\n",
+    "df_records.drop(['ext', 'fn', 'subdir', 'sha256'], axis=1).to_csv(fp_uuids)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 52,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>ext</th>\n",
+       "      <th>fn</th>\n",
+       "      <th>sha256</th>\n",
+       "      <th>subdir</th>\n",
+       "      <th>uuid</th>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>index</th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>jpg</td>\n",
+       "      <td>AJ_Cook_0001</td>\n",
+       "      <td>550937b71b9af36b6083fa1ce7c76e97e3254c439614a6...</td>\n",
+       "      <td>AJ_Cook</td>\n",
+       "      <td>f03fd921-2d56-4e83-8115-f658d6a72287</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>jpg</td>\n",
+       "      <td>AJ_Lamas_0001</td>\n",
+       "      <td>46d7ddeec9b00add61ade2f89277d74e8264a2b6cec193...</td>\n",
+       "      <td>AJ_Lamas</td>\n",
+       "      <td>0c96c5bb-dbd1-4584-bd68-af11664b98bb</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>jpg</td>\n",
+       "      <td>Aaron_Eckhart_0001</td>\n",
+       "      <td>b68ed8d50ba85209d826b962987077bc8e1826f7f2f325...</td>\n",
+       "      <td>Aaron_Eckhart</td>\n",
+       "      <td>8221e75c-9537-4a4f-9693-483b445244b4</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>jpg</td>\n",
+       "      <td>Aaron_Guiel_0001</td>\n",
+       "      <td>156f5428fad30c420ef01d9b0a3ab73e98aa6a1e5a2f0b...</td>\n",
+       "      <td>Aaron_Guiel</td>\n",
+       "      <td>a2955610-ed5e-433c-bdd4-e3a72ff44736</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>jpg</td>\n",
+       "      <td>Aaron_Patterson_0001</td>\n",
+       "      <td>34dfe798220b53aac910e5e39705770d212cdfbe4be8a4...</td>\n",
+       "      <td>Aaron_Patterson</td>\n",
+       "      <td>1d0782e9-ed16-4550-b1e9-d9c03eef6181</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "       ext                    fn  \\\n",
+       "index                              \n",
+       "0      jpg          AJ_Cook_0001   \n",
+       "1      jpg         AJ_Lamas_0001   \n",
+       "2      jpg    Aaron_Eckhart_0001   \n",
+       "3      jpg      Aaron_Guiel_0001   \n",
+       "4      jpg  Aaron_Patterson_0001   \n",
+       "\n",
+       "                                                  sha256           subdir  \\\n",
+       "index                                                                       \n",
+       "0      550937b71b9af36b6083fa1ce7c76e97e3254c439614a6...          AJ_Cook   \n",
+       "1      46d7ddeec9b00add61ade2f89277d74e8264a2b6cec193...         AJ_Lamas   \n",
+       "2      b68ed8d50ba85209d826b962987077bc8e1826f7f2f325...    Aaron_Eckhart   \n",
+       "3      156f5428fad30c420ef01d9b0a3ab73e98aa6a1e5a2f0b...      Aaron_Guiel   \n",
+       "4      34dfe798220b53aac910e5e39705770d212cdfbe4be8a4...  Aaron_Patterson   \n",
+       "\n",
+       "                                       uuid  \n",
+       "index                                        \n",
+       "0      f03fd921-2d56-4e83-8115-f658d6a72287  \n",
+       "1      0c96c5bb-dbd1-4584-bd68-af11664b98bb  \n",
+       "2      8221e75c-9537-4a4f-9693-483b445244b4  \n",
+       "3      a2955610-ed5e-433c-bdd4-e3a72ff44736  \n",
+       "4      1d0782e9-ed16-4550-b1e9-d9c03eef6181  "
+      ]
+     },
+     "execution_count": 52,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_records.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 132,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>description</th>\n",
+       "      <th>gender</th>\n",
+       "      <th>images</th>\n",
+       "      <th>name</th>\n",
+       "      <th>name_kg</th>\n",
+       "      <th>score</th>\n",
+       "      <th>url</th>\n",
+       "      <th>name_new</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Canadian actress</td>\n",
+       "      <td>f</td>\n",
+       "      <td>1</td>\n",
+       "      <td>AJ Cook</td>\n",
+       "      <td>A. J. Cook</td>\n",
+       "      <td>274.554810</td>\n",
+       "      <td>http://www.ajcookofficial.com</td>\n",
+       "      <td></td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>American actor</td>\n",
+       "      <td>m</td>\n",
+       "      <td>1</td>\n",
+       "      <td>AJ Lamas</td>\n",
+       "      <td>AJ Lamas</td>\n",
+       "      <td>389.547211</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td></td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "        description gender  images      name     name_kg       score  \\\n",
+       "0  Canadian actress      f       1   AJ Cook  A. J. Cook  274.554810   \n",
+       "1    American actor      m       1  AJ Lamas    AJ Lamas  389.547211   \n",
+       "\n",
+       "                             url name_new  \n",
+       "0  http://www.ajcookofficial.com           \n",
+       "1                            NaN           "
+      ]
+     },
+     "execution_count": 132,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_kg = pd.read_csv(fp_names_gender_kg)\n",
+    "df_kg_new = df_kg.copy()\n",
+    "df_kg_new['name_new'] = [''] * len(df_kg_new)\n",
+    "df_kg_new.head(2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 133,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>description</th>\n",
+       "      <th>gender</th>\n",
+       "      <th>images</th>\n",
+       "      <th>name</th>\n",
+       "      <th>name_kg</th>\n",
+       "      <th>score</th>\n",
+       "      <th>url</th>\n",
+       "      <th>name_new</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Canadian actress</td>\n",
+       "      <td>f</td>\n",
+       "      <td>1</td>\n",
+       "      <td>AJ Cook</td>\n",
+       "      <td>A. J. Cook</td>\n",
+       "      <td>274.554810</td>\n",
+       "      <td>http://www.ajcookofficial.com</td>\n",
+       "      <td>A. J. Cook</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>American actor</td>\n",
+       "      <td>m</td>\n",
+       "      <td>1</td>\n",
+       "      <td>AJ Lamas</td>\n",
+       "      <td>AJ Lamas</td>\n",
+       "      <td>389.547211</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>AJ Lamas</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "        description gender  images      name     name_kg       score  \\\n",
+       "0  Canadian actress      f       1   AJ Cook  A. J. Cook  274.554810   \n",
+       "1    American actor      m       1  AJ Lamas    AJ Lamas  389.547211   \n",
+       "\n",
+       "                             url    name_new  \n",
+       "0  http://www.ajcookofficial.com  A. J. Cook  \n",
+       "1                            NaN    AJ Lamas  "
+      ]
+     },
+     "execution_count": 133,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# correct original LFW names using\n",
+    "for idx, row in df_kg_new.iterrows():\n",
+    "  name_kg = str(row.get('name_kg', ''))\n",
+    "  name = str(row.get('name', ''))\n",
+    "  name_new = row['name']\n",
+    "  if name_kg:\n",
+    "    sim = similarity(name, name_kg)\n",
+    "    if sim > .75:\n",
+    "      name_new = row['name_kg']\n",
+    "  df_kg_new.at[idx, 'name_new'] = name_new\n",
+    "\n",
+    "df_kg_new.head(2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 136,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_kg_new['index'] = [''] * len(df_kg_new)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 134,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>description</th>\n",
+       "      <th>gender</th>\n",
+       "      <th>images</th>\n",
+       "      <th>name</th>\n",
+       "      <th>name_kg</th>\n",
+       "      <th>name_new</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Canadian actress</td>\n",
+       "      <td>f</td>\n",
+       "      <td>1</td>\n",
+       "      <td>AJ Cook</td>\n",
+       "      <td>A. J. Cook</td>\n",
+       "      <td>A. J. Cook</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>American actor</td>\n",
+       "      <td>m</td>\n",
+       "      <td>1</td>\n",
+       "      <td>AJ Lamas</td>\n",
+       "      <td>AJ Lamas</td>\n",
+       "      <td>AJ Lamas</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "        description gender  images      name     name_kg    name_new\n",
+       "0  Canadian actress      f       1   AJ Cook  A. J. Cook  A. J. Cook\n",
+       "1    American actor      m       1  AJ Lamas    AJ Lamas    AJ Lamas"
+      ]
+     },
+     "execution_count": 134,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# drop extra info\n",
+    "df_kg_new = df_kg_new.drop(['score', 'url'], axis=1)\n",
+    "df_kg_new.head(2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 138,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 5749/5749 [00:05<00:00, 1006.44it/s]\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>description</th>\n",
+       "      <th>gender</th>\n",
+       "      <th>images</th>\n",
+       "      <th>name</th>\n",
+       "      <th>name_kg</th>\n",
+       "      <th>name_new</th>\n",
+       "      <th>index</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Canadian actress</td>\n",
+       "      <td>f</td>\n",
+       "      <td>1</td>\n",
+       "      <td>AJ Cook</td>\n",
+       "      <td>A. J. Cook</td>\n",
+       "      <td>A. J. Cook</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>American actor</td>\n",
+       "      <td>m</td>\n",
+       "      <td>1</td>\n",
+       "      <td>AJ Lamas</td>\n",
+       "      <td>AJ Lamas</td>\n",
+       "      <td>AJ Lamas</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "        description gender  images      name     name_kg    name_new index\n",
+       "0  Canadian actress      f       1   AJ Cook  A. J. Cook  A. J. Cook     0\n",
+       "1    American actor      m       1  AJ Lamas    AJ Lamas    AJ Lamas     1"
+      ]
+     },
+     "execution_count": 138,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# add index column\n",
+    "limit = 100000\n",
+    "for idx, row in tqdm(df_kg_new[:limit].iterrows(), total=len(df_kg_new[:limit])):\n",
+    "  name = row['name']  # original LFW\n",
+    "  # get sha256 from records where match\n",
+    "  subdir = name.replace(' ', '_')\n",
+    "  row_match = df_records.loc[(df_records['subdir'] == subdir)]\n",
+    "  df_kg_new.at[idx, 'index'] = int(row_match.index[0])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 140,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_kg_new = df_kg_new.drop(['name', 'name_kg'], axis=1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 151,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_kg_new = df_kg_new.rename(columns={'name_new': 'name'})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 152,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_kg_new.to_csv(fp_identity_meta, index=False)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Convert ROIs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 145,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>h</th>\n",
+       "      <th>image_height</th>\n",
+       "      <th>image_width</th>\n",
+       "      <th>w</th>\n",
+       "      <th>x</th>\n",
+       "      <th>y</th>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>index</th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0.330000</td>\n",
+       "      <td>250</td>\n",
+       "      <td>250</td>\n",
+       "      <td>0.330000</td>\n",
+       "      <td>0.336667</td>\n",
+       "      <td>0.350000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>0.393333</td>\n",
+       "      <td>250</td>\n",
+       "      <td>250</td>\n",
+       "      <td>0.393333</td>\n",
+       "      <td>0.286667</td>\n",
+       "      <td>0.313333</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "              h  image_height  image_width         w         x         y\n",
+       "index                                                                   \n",
+       "0      0.330000           250          250  0.330000  0.336667  0.350000\n",
+       "1      0.393333           250          250  0.393333  0.286667  0.313333"
+      ]
+     },
+     "execution_count": 145,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_rois = pd.read_csv(fp_rois).set_index('index')\n",
+    "df_rois.head(2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 146,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_index = pd.read_csv(fp_index)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 166,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#row = df_records.iloc[ (df_records)]\n",
+    "row = df_kg_new.loc[(df_kg_new['name'] == 'B.B. King')]\n",
+    "#print('index', row['index'].index[0])\n",
+    "row = df_index.iloc[row['index']]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 171,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "7cf753f9e1256e433901a262030f4d184afb4002b49e6b1c7a2d59d07306c2ff\n"
+     ]
+    }
+   ],
+   "source": [
+    "sha = row['sha256']\n",
+    "print(sha.values[0])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 14399/14399 [00:15<00:00, 914.63it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "limit = 10\n",
+    "for idx, row in tqdm(df_kg[:limt].iterrows(), total=len(df_kg[:limt])):\n",
+    "  fn = row['fn']\n",
+    "  subdir = row['subdir']\n",
+    "  # get sha256 from records where match\n",
+    "  row_match = df_records.loc[(df_records['subdir'] == subdir)]\n",
+    "  df_records.at[idx, 'idx'] = int(row_match.index[0])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_identity_meta = df_identity_meta.drop(['ext', 'url', 'score'], axis=1)\n",
+    "df_identity_meta.to_csv(fp_identity_meta)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python [conda env:megapixels]",
+   "language": "python",
+   "name": "conda-env-megapixels-py"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}