summaryrefslogtreecommitdiff
path: root/megapixels/notebooks/datasets/vgg_face2/clean_vgg_02_dask.ipynb
diff options
context:
space:
mode:
Diffstat (limited to 'megapixels/notebooks/datasets/vgg_face2/clean_vgg_02_dask.ipynb')
-rw-r--r--megapixels/notebooks/datasets/vgg_face2/clean_vgg_02_dask.ipynb464
1 files changed, 464 insertions, 0 deletions
diff --git a/megapixels/notebooks/datasets/vgg_face2/clean_vgg_02_dask.ipynb b/megapixels/notebooks/datasets/vgg_face2/clean_vgg_02_dask.ipynb
new file mode 100644
index 00000000..6477d89f
--- /dev/null
+++ b/megapixels/notebooks/datasets/vgg_face2/clean_vgg_02_dask.ipynb
@@ -0,0 +1,464 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Append UUID to SHA256 CSV"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from os.path import join\n",
+ "from pathlib import Path\n",
+ "import difflib\n",
+ "\n",
+ "from tqdm import tqdm_notebook as tqdm\n",
+ "import pandas as pd\n",
+ "import dask.dataframe as dd\n",
+ "import numpy as np"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# names\n",
+ "DATA_STORE_NAS = '/data_store_nas/'\n",
+ "dir_dataset = 'datasets/people/vgg_face2/metadata'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 25,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>sha256</th>\n",
+ " <th>identity</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>index</th>\n",
+ " <th></th>\n",
+ " <th></th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>a39a1df855cb0c70dc553c5e9afa35b4f7c00f4011ca10...</td>\n",
+ " <td>-1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>e360f93613baa68cede6731d2603873cdabd3993841cfd...</td>\n",
+ " <td>-1</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " sha256 identity\n",
+ "index \n",
+ "0 a39a1df855cb0c70dc553c5e9afa35b4f7c00f4011ca10... -1\n",
+ "1 e360f93613baa68cede6731d2603873cdabd3993841cfd... -1"
+ ]
+ },
+ "execution_count": 25,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# associate each file with an identity\n",
+ "fp_index = join(DATA_STORE_NAS, dir_dataset, 'index.csv')\n",
+ "df_index = pd.read_csv(fp_index).set_index('index')\n",
+ "df_index['identity'] = [-1] * len(df_index)\n",
+ "df_index.head(2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 26,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "3311286\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>ext</th>\n",
+ " <th>fn</th>\n",
+ " <th>subdir</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>index</th>\n",
+ " <th></th>\n",
+ " <th></th>\n",
+ " <th></th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>jpg</td>\n",
+ " <td>0089_01</td>\n",
+ " <td>test/n006211</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>jpg</td>\n",
+ " <td>0168_01</td>\n",
+ " <td>test/n006211</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2</th>\n",
+ " <td>jpg</td>\n",
+ " <td>0213_01</td>\n",
+ " <td>test/n006211</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>3</th>\n",
+ " <td>jpg</td>\n",
+ " <td>0010_01</td>\n",
+ " <td>test/n006211</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4</th>\n",
+ " <td>jpg</td>\n",
+ " <td>0115_01</td>\n",
+ " <td>test/n006211</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " ext fn subdir\n",
+ "index \n",
+ "0 jpg 0089_01 test/n006211\n",
+ "1 jpg 0168_01 test/n006211\n",
+ "2 jpg 0213_01 test/n006211\n",
+ "3 jpg 0010_01 test/n006211\n",
+ "4 jpg 0115_01 test/n006211"
+ ]
+ },
+ "execution_count": 26,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# get file info\n",
+ "fp_files = join(DATA_STORE_NAS, dir_dataset, 'files.csv')\n",
+ "df_files = pd.read_csv(fp_files).set_index('index')\n",
+ "print(len(df_files))\n",
+ "df_files.head()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "9131\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>class_id</th>\n",
+ " <th>description</th>\n",
+ " <th>gender</th>\n",
+ " <th>images</th>\n",
+ " <th>name</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>index</th>\n",
+ " <th></th>\n",
+ " <th></th>\n",
+ " <th></th>\n",
+ " <th></th>\n",
+ " <th></th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>n000001</td>\n",
+ " <td>Dalai Lama</td>\n",
+ " <td>m</td>\n",
+ " <td>424</td>\n",
+ " <td>14th Dalai Lama</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>n000002</td>\n",
+ " <td>American singer-songwriter</td>\n",
+ " <td>f</td>\n",
+ " <td>315</td>\n",
+ " <td>A Fine Frenzy</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " class_id description gender images name\n",
+ "index \n",
+ "0 n000001 Dalai Lama m 424 14th Dalai Lama\n",
+ "1 n000002 American singer-songwriter f 315 A Fine Frenzy"
+ ]
+ },
+ "execution_count": 27,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "fp_identities = join(DATA_STORE_NAS, dir_dataset, 'identities.csv')\n",
+ "df_identities = dd.read_csv(fp_identities).set_index('index')\n",
+ "print(len(df_identities))\n",
+ "df_identities.head(2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 28,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "class_ids = {}\n",
+ "for row in df_identities.itertuples():\n",
+ " class_id = row.class_id\n",
+ " index = row.Index\n",
+ " if class_id not in class_ids.keys():\n",
+ " class_ids[class_id] = index"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 31,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "ac332e87bee54d53a0e29efbdfa86d65",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "HBox(children=(IntProgress(value=0, max=3311286), HTML(value='')))"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "for row in tqdm(df_index.itertuples(), total=len(df_index)):\n",
+ " file_index = row.Index\n",
+ " file_row = df_files.iloc[file_index]\n",
+ " subdir = file_row['subdir']\n",
+ " class_id = subdir.split('/')[1]\n",
+ " identity_index = class_ids[class_id]\n",
+ " df_index.at[row.Index, 'identity'] = identity_index"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 34,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "fp_index_new = join(DATA_STORE_NAS, dir_dataset, 'index_new.csv')\n",
+ "df_index.to_csv(fp_index_new)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 30,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>sha256</th>\n",
+ " <th>identity</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>index</th>\n",
+ " <th></th>\n",
+ " <th></th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>a39a1df855cb0c70dc553c5e9afa35b4f7c00f4011ca10...</td>\n",
+ " <td>6123</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>e360f93613baa68cede6731d2603873cdabd3993841cfd...</td>\n",
+ " <td>6123</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2</th>\n",
+ " <td>3920a8bdf523a5bf7da9258ec414a741462d0cfbec8d2c...</td>\n",
+ " <td>6123</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>3</th>\n",
+ " <td>577ce218e4a61e612942c55fd172cac4b48becacbfc708...</td>\n",
+ " <td>6123</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>4</th>\n",
+ " <td>b27d37425a4e59dc4d37c3df331d0b69e4919338a3d46f...</td>\n",
+ " <td>6123</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " sha256 identity\n",
+ "index \n",
+ "0 a39a1df855cb0c70dc553c5e9afa35b4f7c00f4011ca10... 6123\n",
+ "1 e360f93613baa68cede6731d2603873cdabd3993841cfd... 6123\n",
+ "2 3920a8bdf523a5bf7da9258ec414a741462d0cfbec8d2c... 6123\n",
+ "3 577ce218e4a61e612942c55fd172cac4b48becacbfc708... 6123\n",
+ "4 b27d37425a4e59dc4d37c3df331d0b69e4919338a3d46f... 6123"
+ ]
+ },
+ "execution_count": 30,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "df_index.head()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python [conda env:megapixels]",
+ "language": "python",
+ "name": "conda-env-megapixels-py"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.6.6"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
generated by cgit v1.2.3-70-g09d2 (git 2.51.0) at 2025-12-10 19:59:22 +0000