From a16b3cc7f796a5abe6c8c79f22b178785e6971f5 Mon Sep 17 00:00:00 2001 From: adamhrv Date: Tue, 19 Mar 2019 12:21:21 +0100 Subject: nbs for data collection --- .../datasets/ibm_dif/create_name_list.ipynb | 141 ++++++++++ .../notebooks/datasets/ibm_dif/decrypt.ipynb | 308 +++++++++++++++++++++ .../notebooks/datasets/ibm_dif/username_list.ipynb | 240 ++++++++++++++++ 3 files changed, 689 insertions(+) create mode 100644 megapixels/notebooks/datasets/ibm_dif/create_name_list.ipynb create mode 100644 megapixels/notebooks/datasets/ibm_dif/decrypt.ipynb create mode 100644 megapixels/notebooks/datasets/ibm_dif/username_list.ipynb (limited to 'megapixels/notebooks/datasets/ibm_dif') diff --git a/megapixels/notebooks/datasets/ibm_dif/create_name_list.ipynb b/megapixels/notebooks/datasets/ibm_dif/create_name_list.ipynb new file mode 100644 index 00000000..c3484f41 --- /dev/null +++ b/megapixels/notebooks/datasets/ibm_dif/create_name_list.ipynb @@ -0,0 +1,141 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Create Name List" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "%reload_ext autoreload\n", + "%autoreload 2\n", + "\n", + "import os\n", + "from os.path import join\n", + "from glob import glob\n", + "from pathlib import Path\n", + "import requests\n", + "import json\n", + "from urllib.parse import unquote\n", + "\n", + "from tqdm import tqdm_notebook as tqdm\n", + "import pandas as pd\n", + "import numpy as np\n", + "\n", + "import sys\n", + "sys.path.append('/work/megapixels_dev/megapixels')\n", + "from app.utils import api_utils, identity_utils\n", + "from app.settings import app_cfg\n", + "from app.settings import types" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "fp_yfcc = '/data_store_hdd/datasets/people/yfcc100m/downloads/usernames.txt'\n", + "fp_yfcc_clean = '/data_store_hdd/datasets/people/yfcc100m/downloads/usernames_clean.txt'\n", + "fp_dir_research = '/data_store_hdd/datasets/people/ibm_dif/research/'\n", + "fp_usernames = join(fp_dir_research, 'usernames_decoded.csv')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Convert YFCC to Unique Name List" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "with open(fp_yfcc, 'r') as fp:\n", + " usernames_raw = fp.readlines()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ea6b4e8d2a8145f2b7efb559575862b8", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, max=100000000), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "usernames = []\n", + "for username in tqdm(usernames_raw):\n", + " un_clean = unquote(username.strip().lower().replace(' ', '+'))\n", + " usernames.append(un_clean)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "with open(fp_yfcc_clean, 'w') as fp:\n", + " for username in usernames:\n", + " fp.write(username + \"\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "megapixels", + "language": "python", + "name": "megapixels" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/megapixels/notebooks/datasets/ibm_dif/decrypt.ipynb b/megapixels/notebooks/datasets/ibm_dif/decrypt.ipynb new file mode 100644 index 00000000..bdd7d6a2 --- /dev/null +++ b/megapixels/notebooks/datasets/ibm_dif/decrypt.ipynb @@ -0,0 +1,308 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Decrypt IBM" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "%reload_ext autoreload\n", + "%autoreload 2\n", + "\n", + "import os\n", + "from os.path import join\n", + "from glob import glob\n", + "from pathlib import Path\n", + "import requests\n", + "import json\n", + "from pprint import pprint\n", + "from multiprocessing.pool import ThreadPool\n", + "import threading\n", + "import urllib.request\n", + "import hashlib\n", + "import subprocess\n", + "\n", + "from tqdm import tqdm_notebook as tqdm\n", + "import pandas as pd\n", + "import numpy as np\n", + "from random import randint\n", + "\n", + "import sys\n", + "sys.path.append('/work/megapixels_dev/megapixels')\n", + "from app.utils import api_utils, identity_utils\n", + "from app.settings import app_cfg\n", + "from app.settings import types" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "fp_dir_research = '/data_store_hdd/datasets/people/ibm_dif/research/'\n", + "fp_usernames = join(fp_dir_research, 'username_sha.csv')\n", + "dir_json = '/data_store_hdd/datasets/people/ibm_dif/research/valid_files/'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Decrypt" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "def decrypt(username, shaval):\n", + " cmd = ['/home/adam/.nvm/versions/node/v9.9.0/bin/node', \n", + " '/data_store/datasets/people/ibm_dif/web_files/decrypt_cli', \n", + " username,\n", + " shaval]\n", + " proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n", + " o, e = proc.communicate()\n", + " return json.loads(o.decode())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "a = {'a':1, 'b':2}\n", + "c = {'c':3}\n" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "df_usernames = pd.read_csv(fp_usernames)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "records = df_usernames.to_dict('records')" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "96414 581092\n" + ] + } + ], + "source": [ + "print(len(matched_records), len(records))" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2b2f471839d347f1b7df78698352bee6", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, max=581092), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "# create list of valid files\n", + "matched_records = []\n", + "\n", + "for record in tqdm(records):\n", + " shaval = record['sha256']\n", + " username = record['username']\n", + " fp_in = join(dir_json, shaval + '.json')\n", + " if Path(fp_in).is_file():\n", + " matched_records.append({'sha256':shaval, 'username': username})" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "48931\n" + ] + } + ], + "source": [ + "print(len(matched_records))" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "87b24eb93ea24736b8a2f429c97a82f2", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, max=48931), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Error: Expecting value: line 1 column 1 (char 0)\n", + "Error: Expecting value: line 1 column 1 (char 0)\n", + "Error: Expecting value: line 1 column 1 (char 0)\n", + "Error: Expecting value: line 1 column 1 (char 0)\n", + "Error: Expecting value: line 1 column 1 (char 0)\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mPath\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfp_in\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_file\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdecrypt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0musername\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshaval\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 8\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m'sha256'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mshaval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'username'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0musername\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'count'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'count'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'url'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'url'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0mmatched_records\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m\u001b[0m in \u001b[0;36mdecrypt\u001b[0;34m(username, shaval)\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0musername\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m shaval]\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mproc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msubprocess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcmd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstdout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msubprocess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPIPE\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstderr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msubprocess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPIPE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 7\u001b[0m \u001b[0mo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0me\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mproc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcommunicate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/envs/megapixels/lib/python3.6/subprocess.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors)\u001b[0m\n\u001b[1;32m 727\u001b[0m \u001b[0mc2pread\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mc2pwrite\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 728\u001b[0m \u001b[0merrread\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrwrite\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 729\u001b[0;31m restore_signals, start_new_session)\n\u001b[0m\u001b[1;32m 730\u001b[0m \u001b[0;32mexcept\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 731\u001b[0m \u001b[0;31m# Cleanup if the child failed starting.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/envs/megapixels/lib/python3.6/subprocess.py\u001b[0m in \u001b[0;36m_execute_child\u001b[0;34m(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session)\u001b[0m\n\u001b[1;32m 1293\u001b[0m \u001b[0merrread\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrwrite\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1294\u001b[0m \u001b[0merrpipe_read\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrpipe_write\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1295\u001b[0;31m restore_signals, start_new_session, preexec_fn)\n\u001b[0m\u001b[1;32m 1296\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_child_created\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1297\u001b[0m \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "for record in tqdm(matched_records):\n", + " if record['url'] and record['count']:\n", + " continue\n", + " shaval = record['sha256']\n", + " username = record['username']\n", + " fp_in = join(dir_json, shaval + '.json')\n", + " if Path(fp_in).is_file():\n", + " try:\n", + " result = decrypt(username, shaval)\n", + " obj = {'sha256':shaval, 'username': username, 'count': result['count'], 'url': result['url']}\n", + " matched_records.append(obj)\n", + " except Exception as e:\n", + " print('Error:', e)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_records = pd.DataFrame.from_dict(matched_records)\n", + "df_records.to_csv('/data_store_hdd/datasets/people/ibm_dif/research/ibm_dif_metadata.csv', index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# make a CSV for downloading\n", + "filemaps = []\n", + "for item in matched_records:\n", + " fp = join(Path(a).parent.name, Path(a).name)\n", + " obj = {'filepath': fp, 'url': item['url']}\n", + " filemaps.append(obj)\n", + "df_filemaps = pd.DataFrame.from_dict(filemaps)\n", + "df_filemaps.to_csv('/data_store_hdd/datasets/people/ibm_dif/research/ibm_dif_urls.csv', index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# working example\n", + "username = 'lknobel'\n", + "shaval = '00036fc3de043beca0c00a88d9d5cec4853a2d9a42c9c1ffdbb0e1f7431c488d'\n", + "result = decrypt(username, shaval)\n", + "print(result['count'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "megapixels", + "language": "python", + "name": "megapixels" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/megapixels/notebooks/datasets/ibm_dif/username_list.ipynb b/megapixels/notebooks/datasets/ibm_dif/username_list.ipynb new file mode 100644 index 00000000..36cb8339 --- /dev/null +++ b/megapixels/notebooks/datasets/ibm_dif/username_list.ipynb @@ -0,0 +1,240 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Generate sha256,username list" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "%reload_ext autoreload\n", + "%autoreload 2\n", + "\n", + "import os\n", + "from os.path import join\n", + "from glob import glob\n", + "from pathlib import Path\n", + "import requests\n", + "import json\n", + "from pprint import pprint\n", + "from multiprocessing.pool import ThreadPool\n", + "import threading\n", + "import urllib.request\n", + "import hashlib\n", + "\n", + "from tqdm import tqdm_notebook as tqdm\n", + "import pandas as pd\n", + "import numpy as np\n", + "from random import randint\n", + "\n", + "import sys\n", + "sys.path.append('/work/megapixels_dev/megapixels')\n", + "from app.utils import api_utils, identity_utils\n", + "\n", + "from app.settings import app_cfg\n", + "from app.settings import types" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "fp_in = '/data_store_hdd/datasets/people/yfcc100m/downloads/usernames_clean.txt'\n", + "fp_10k_in = '/data_store_hdd/datasets/people/yfcc100m/downloads/usernames_clean_10k.txt'\n", + "fp_out = '/data_store_hdd/datasets/people/ibm_dif/research/username_sha.csv'" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "with open(fp_in, 'r') as fp:\n", + " usernames = fp.readlines()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "usernames = [x.strip() for x in usernames]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Convert usernames to SHA256" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "metadataset = []" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f2171f3e48eb4a71b7c8691169aefbcf", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, max=100000000), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "for username in tqdm(usernames):\n", + " shaname = hashlib.sha256(str.encode(username)).hexdigest()\n", + " metadataset.append({'username': username, 'sha256': shaname})" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "del usernames\n", + "df_users = pd.DataFrame.from_dict(metadataset)\n", + "del metadataset\n", + "df_users = df_users.drop_duplicates(subset='sha256', keep=\"last\")\n", + "df_users.to_csv(fp_out, index=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create download CSV" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "fp_in = '/data_store_hdd/datasets/people/ibm_dif/research/ibm_dif_metadata.csv'\n", + "fp_out = '/data_store_hdd/datasets/people/ibm_dif/research/ibm_dif_urls.csv'" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "records = pd.read_csv(fp_in).to_dict('records')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# http://farm5.staticflickr.com/4086/4993389409_f1140639d5.jpg,old+guard+museum" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "filepaths = []\n", + "for record in records:\n", + " filepaths.append(Path(record['url']).name)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "url_maps = []\n", + "for record in records:\n", + " url_maps.append({'url': record['url'], 'filepath': Path(record['url']).name})" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "df_urls = pd.DataFrame.from_dict(url_maps)" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "df_urls.to_csv(fp_out, index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "megapixels", + "language": "python", + "name": "megapixels" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} -- cgit v1.2.3-70-g09d2