nbs for data collection

3 files changed, 689 insertions, 0 deletions

diff --git a/megapixels/notebooks/datasets/ibm_dif/create_name_list.ipynb b/megapixels/notebooks/datasets/ibm_dif/create_name_list.ipynb
new file mode 100644
index 00000000..c3484f41
--- /dev/null
+++ b/megapixels/notebooks/datasets/ibm_dif/create_name_list.ipynb
@@ -0,0 +1,141 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Create Name List"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%reload_ext autoreload\n",
+    "%autoreload 2\n",
+    "\n",
+    "import os\n",
+    "from os.path import join\n",
+    "from glob import glob\n",
+    "from pathlib import Path\n",
+    "import requests\n",
+    "import json\n",
+    "from urllib.parse import unquote\n",
+    "\n",
+    "from tqdm import tqdm_notebook as tqdm\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "\n",
+    "import sys\n",
+    "sys.path.append('/work/megapixels_dev/megapixels')\n",
+    "from app.utils import api_utils, identity_utils\n",
+    "from app.settings import app_cfg\n",
+    "from app.settings import types"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fp_yfcc = '/data_store_hdd/datasets/people/yfcc100m/downloads/usernames.txt'\n",
+    "fp_yfcc_clean = '/data_store_hdd/datasets/people/yfcc100m/downloads/usernames_clean.txt'\n",
+    "fp_dir_research = '/data_store_hdd/datasets/people/ibm_dif/research/'\n",
+    "fp_usernames = join(fp_dir_research, 'usernames_decoded.csv')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Convert YFCC to Unique Name List"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open(fp_yfcc, 'r') as fp:\n",
+    "  usernames_raw = fp.readlines()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ea6b4e8d2a8145f2b7efb559575862b8",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, max=100000000), HTML(value='')))"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "usernames = []\n",
+    "for username in tqdm(usernames_raw):\n",
+    "  un_clean = unquote(username.strip().lower().replace(' ', '+'))\n",
+    "  usernames.append(un_clean)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open(fp_yfcc_clean, 'w') as fp:\n",
+    "  for username in usernames:\n",
+    "    fp.write(username + \"\\n\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "megapixels",
+   "language": "python",
+   "name": "megapixels"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/megapixels/notebooks/datasets/ibm_dif/decrypt.ipynb b/megapixels/notebooks/datasets/ibm_dif/decrypt.ipynb
new file mode 100644
index 00000000..bdd7d6a2
--- /dev/null
+++ b/megapixels/notebooks/datasets/ibm_dif/decrypt.ipynb
@@ -0,0 +1,308 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Decrypt IBM"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%reload_ext autoreload\n",
+    "%autoreload 2\n",
+    "\n",
+    "import os\n",
+    "from os.path import join\n",
+    "from glob import glob\n",
+    "from pathlib import Path\n",
+    "import requests\n",
+    "import json\n",
+    "from pprint import pprint\n",
+    "from multiprocessing.pool import ThreadPool\n",
+    "import threading\n",
+    "import urllib.request\n",
+    "import hashlib\n",
+    "import subprocess\n",
+    "\n",
+    "from tqdm import tqdm_notebook as tqdm\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "from random import randint\n",
+    "\n",
+    "import sys\n",
+    "sys.path.append('/work/megapixels_dev/megapixels')\n",
+    "from app.utils import api_utils, identity_utils\n",
+    "from app.settings import app_cfg\n",
+    "from app.settings import types"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fp_dir_research = '/data_store_hdd/datasets/people/ibm_dif/research/'\n",
+    "fp_usernames = join(fp_dir_research, 'username_sha.csv')\n",
+    "dir_json = '/data_store_hdd/datasets/people/ibm_dif/research/valid_files/'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Decrypt"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def decrypt(username, shaval):\n",
+    "  cmd = ['/home/adam/.nvm/versions/node/v9.9.0/bin/node', \n",
+    "       '/data_store/datasets/people/ibm_dif/web_files/decrypt_cli', \n",
+    "       username,\n",
+    "      shaval]\n",
+    "  proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n",
+    "  o, e = proc.communicate()\n",
+    "  return json.loads(o.decode())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "a = {'a':1, 'b':2}\n",
+    "c = {'c':3}\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_usernames = pd.read_csv(fp_usernames)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "records = df_usernames.to_dict('records')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "96414 581092\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(len(matched_records), len(records))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "2b2f471839d347f1b7df78698352bee6",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, max=581092), HTML(value='')))"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "# create list of valid files\n",
+    "matched_records = []\n",
+    "\n",
+    "for record in tqdm(records):\n",
+    "  shaval = record['sha256']\n",
+    "  username = record['username']\n",
+    "  fp_in = join(dir_json, shaval + '.json')\n",
+    "  if Path(fp_in).is_file():\n",
+    "      matched_records.append({'sha256':shaval, 'username': username})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "48931\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(len(matched_records))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "87b24eb93ea24736b8a2f429c97a82f2",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, max=48931), HTML(value='')))"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Error: Expecting value: line 1 column 1 (char 0)\n",
+      "Error: Expecting value: line 1 column 1 (char 0)\n",
+      "Error: Expecting value: line 1 column 1 (char 0)\n",
+      "Error: Expecting value: line 1 column 1 (char 0)\n",
+      "Error: Expecting value: line 1 column 1 (char 0)\n"
+     ]
+    },
+    {
+     "ename": "KeyboardInterrupt",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
+      "\u001b[0;32m<ipython-input-27-d75b162406b7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0mPath\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfp_in\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_file\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m       \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdecrypt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0musername\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshaval\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m       \u001b[0mobj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m'sha256'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mshaval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'username'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0musername\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'count'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'count'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'url'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'url'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m       \u001b[0mmatched_records\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m<ipython-input-21-db29687a08b0>\u001b[0m in \u001b[0;36mdecrypt\u001b[0;34m(username, shaval)\u001b[0m\n\u001b[1;32m      4\u001b[0m        \u001b[0musername\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m       shaval]\n\u001b[0;32m----> 6\u001b[0;31m   \u001b[0mproc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msubprocess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcmd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstdout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msubprocess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPIPE\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstderr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msubprocess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPIPE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m   \u001b[0mo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0me\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mproc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcommunicate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/anaconda3/envs/megapixels/lib/python3.6/subprocess.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors)\u001b[0m\n\u001b[1;32m    727\u001b[0m                                 \u001b[0mc2pread\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mc2pwrite\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    728\u001b[0m                                 \u001b[0merrread\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrwrite\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 729\u001b[0;31m                                 restore_signals, start_new_session)\n\u001b[0m\u001b[1;32m    730\u001b[0m         \u001b[0;32mexcept\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    731\u001b[0m             \u001b[0;31m# Cleanup if the child failed starting.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/anaconda3/envs/megapixels/lib/python3.6/subprocess.py\u001b[0m in \u001b[0;36m_execute_child\u001b[0;34m(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session)\u001b[0m\n\u001b[1;32m   1293\u001b[0m                             \u001b[0merrread\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrwrite\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1294\u001b[0m                             \u001b[0merrpipe_read\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrpipe_write\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1295\u001b[0;31m                             restore_signals, start_new_session, preexec_fn)\n\u001b[0m\u001b[1;32m   1296\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_child_created\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1297\u001b[0m                 \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
+     ]
+    }
+   ],
+   "source": [
+    "for record in tqdm(matched_records):\n",
+    "  if record['url'] and record['count']:\n",
+    "    continue\n",
+    "  shaval = record['sha256']\n",
+    "  username = record['username']\n",
+    "  fp_in = join(dir_json, shaval + '.json')\n",
+    "  if Path(fp_in).is_file():\n",
+    "    try:\n",
+    "      result = decrypt(username, shaval)\n",
+    "      obj = {'sha256':shaval, 'username': username, 'count': result['count'], 'url': result['url']}\n",
+    "      matched_records.append(obj)\n",
+    "    except Exception as e:\n",
+    "      print('Error:', e)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_records = pd.DataFrame.from_dict(matched_records)\n",
+    "df_records.to_csv('/data_store_hdd/datasets/people/ibm_dif/research/ibm_dif_metadata.csv', index=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# make a CSV for downloading\n",
+    "filemaps = []\n",
+    "for item in matched_records:\n",
+    "  fp = join(Path(a).parent.name, Path(a).name)\n",
+    "  obj = {'filepath': fp, 'url': item['url']}\n",
+    "  filemaps.append(obj)\n",
+    "df_filemaps = pd.DataFrame.from_dict(filemaps)\n",
+    "df_filemaps.to_csv('/data_store_hdd/datasets/people/ibm_dif/research/ibm_dif_urls.csv', index=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# working example\n",
+    "username = 'lknobel'\n",
+    "shaval = '00036fc3de043beca0c00a88d9d5cec4853a2d9a42c9c1ffdbb0e1f7431c488d'\n",
+    "result = decrypt(username, shaval)\n",
+    "print(result['count'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "megapixels",
+   "language": "python",
+   "name": "megapixels"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/megapixels/notebooks/datasets/ibm_dif/username_list.ipynb b/megapixels/notebooks/datasets/ibm_dif/username_list.ipynb
new file mode 100644
index 00000000..36cb8339
--- /dev/null
+++ b/megapixels/notebooks/datasets/ibm_dif/username_list.ipynb
@@ -0,0 +1,240 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Generate sha256,username list"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%reload_ext autoreload\n",
+    "%autoreload 2\n",
+    "\n",
+    "import os\n",
+    "from os.path import join\n",
+    "from glob import glob\n",
+    "from pathlib import Path\n",
+    "import requests\n",
+    "import json\n",
+    "from pprint import pprint\n",
+    "from multiprocessing.pool import ThreadPool\n",
+    "import threading\n",
+    "import urllib.request\n",
+    "import hashlib\n",
+    "\n",
+    "from tqdm import tqdm_notebook as tqdm\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "from random import randint\n",
+    "\n",
+    "import sys\n",
+    "sys.path.append('/work/megapixels_dev/megapixels')\n",
+    "from app.utils import api_utils, identity_utils\n",
+    "\n",
+    "from app.settings import app_cfg\n",
+    "from app.settings import types"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fp_in = '/data_store_hdd/datasets/people/yfcc100m/downloads/usernames_clean.txt'\n",
+    "fp_10k_in = '/data_store_hdd/datasets/people/yfcc100m/downloads/usernames_clean_10k.txt'\n",
+    "fp_out = '/data_store_hdd/datasets/people/ibm_dif/research/username_sha.csv'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open(fp_in, 'r') as fp:\n",
+    "  usernames = fp.readlines()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "usernames = [x.strip() for x in usernames]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Convert usernames to SHA256"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "metadataset = []"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f2171f3e48eb4a71b7c8691169aefbcf",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, max=100000000), HTML(value='')))"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "for username in tqdm(usernames):\n",
+    "  shaname = hashlib.sha256(str.encode(username)).hexdigest()\n",
+    "  metadataset.append({'username': username, 'sha256': shaname})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "del usernames\n",
+    "df_users = pd.DataFrame.from_dict(metadataset)\n",
+    "del metadataset\n",
+    "df_users = df_users.drop_duplicates(subset='sha256', keep=\"last\")\n",
+    "df_users.to_csv(fp_out, index=False)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Create download CSV"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fp_in = '/data_store_hdd/datasets/people/ibm_dif/research/ibm_dif_metadata.csv'\n",
+    "fp_out = '/data_store_hdd/datasets/people/ibm_dif/research/ibm_dif_urls.csv'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "records = pd.read_csv(fp_in).to_dict('records')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# http://farm5.staticflickr.com/4086/4993389409_f1140639d5.jpg,old+guard+museum"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "filepaths = []\n",
+    "for record in records:\n",
+    "  filepaths.append(Path(record['url']).name)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "url_maps = []\n",
+    "for record in records:\n",
+    "  url_maps.append({'url': record['url'], 'filepath': Path(record['url']).name})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_urls = pd.DataFrame.from_dict(url_maps)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_urls.to_csv(fp_out, index=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "megapixels",
+   "language": "python",
+   "name": "megapixels"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}