add MSC name scraper nb

1 files changed, 189 insertions, 0 deletions

diff --git a/megapixels/notebooks/datasets/munich_security_conference/html2csv.ipynb b/megapixels/notebooks/datasets/munich_security_conference/html2csv.ipynb
new file mode 100644
index 00000000..aa819214
--- /dev/null
+++ b/megapixels/notebooks/datasets/munich_security_conference/html2csv.ipynb
@@ -0,0 +1,189 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Convert MSC HTML to CSV\n",
+    "\n",
+    "- create name lists"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%reload_ext autoreload\n",
+    "%autoreload 2\n",
+    "\n",
+    "from os.path import join\n",
+    "from pathlib import Path\n",
+    "from functools import partial\n",
+    "from multiprocessing.dummy import Pool as ThreadPool\n",
+    "\n",
+    "import lxml\n",
+    "from bs4 import BeautifulSoup\n",
+    "import urllib.request\n",
+    "from tqdm import tqdm_notebook as tqdm\n",
+    "import pandas as pd\n",
+    "\n",
+    "import sys\n",
+    "sys.path.append('/work/megapixels_dev/megapixels/')\n",
+    "from app.settings import app_cfg as cfg\n",
+    "from app.utils import file_utils, im_utils"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 147,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fp_dir_in = '/data_store/datasets/munich_security_conference/participants/'\n",
+    "fp_dir_out = '/data_store/datasets/munich_security_conference/participants/'\n",
+    "fp_out_all_csv = join(fp_dir_ou, 'participants.csv')  # total list\n",
+    "years = ['2009', '2010', '2011', '2014']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 143,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def parse_name(name):\n",
+    "  try:\n",
+    "    ridx = name.rindex(',')\n",
+    "  except Exception as e:\n",
+    "    # names like \"Ban Ki-moon\" have no comman\n",
+    "    if name == 'Ban Ki-moon':\n",
+    "      name = 'Ki-moon, Ban'\n",
+    "    elif name == 'Fu Ying':\n",
+    "      name = 'Ying, Fu'\n",
+    "    elif name == 'Dr. Ng Eng Hen':\n",
+    "      # unclear: https://en.wikipedia.org/wiki/Ng_Eng_Hen\n",
+    "      name = 'Ng, Dr. Eng Hen' \n",
+    "    elif name == 'Seok-soo Lee':\n",
+    "      name = 'Lee, Seok-soo'\n",
+    "    else:\n",
+    "      print(f'Could not handle: \"{name}\"')\n",
+    "    ridx = name.rindex(',')\n",
+    "    \n",
+    "  name_last = name[:ridx].strip()\n",
+    "  name_first = name[(ridx + 1):].strip()\n",
+    "  return name_first, name_last\n",
+    "  \n",
+    "def parse_year(fp_in_html, year):\n",
+    "  # create soup\n",
+    "  with open(fp_in_html, 'r') as fp:\n",
+    "    data = fp.read()\n",
+    "  soup = BeautifulSoup(data, 'lxml')\n",
+    "  \n",
+    "  # get rows\n",
+    "  table = soup.find('table', attrs={'class':'contenttable'})\n",
+    "  tbody = table.find('tbody')\n",
+    "  trows = tbody.find_all('tr')\n",
+    "  \n",
+    "  # parse by year\n",
+    "  participants = []\n",
+    "  for trow in trows[1:]:\n",
+    "    if year == '2009' or year == '2014':\n",
+    "      tds = trow.find_all('td')\n",
+    "      name = tds[0].text.strip()\n",
+    "      name_first, name_last = parse_name(name)\n",
+    "      desc = tds[1].text.strip()\n",
+    "    elif year == '2010':\n",
+    "      tds = trow.find_all('td')\n",
+    "      name_first = tds[0].text.strip()\n",
+    "      name_last = tds[1].text.strip()\n",
+    "      desc = tds[2].text.strip()\n",
+    "    elif year == '2011':\n",
+    "      tds = trow.find_all('td')\n",
+    "      name = tds[0].find_all('p')[0].text.strip()\n",
+    "      name_first, name_last = parse_name(name)\n",
+    "      desc = tds[1].find_all('p')[0].text.strip()\n",
+    "      \n",
+    "    obj = {'name_first': name_first, 'name_last': name_last, 'description': desc, 'year': year}\n",
+    "    participants.append(obj)\n",
+    "    \n",
+    "  return participants"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 148,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2009\n",
+      "Wrote: /data_store/datasets/munich_security_conference/participants/2009.csv with 346 items\n",
+      "2010\n",
+      "Wrote: /data_store/datasets/munich_security_conference/participants/2010.csv with 317 items\n",
+      "2011\n",
+      "Wrote: /data_store/datasets/munich_security_conference/participants/2011.csv with 341 items\n",
+      "2014\n",
+      "Wrote: /data_store/datasets/munich_security_conference/participants/2014.csv with 467 items\n",
+      "Wrote: /data_store/datasets/munich_security_conference/participants/participants.csv with 1471 items\n"
+     ]
+    }
+   ],
+   "source": [
+    "participants_all = []\n",
+    "for year in years:\n",
+    "  fp_in_html = join(fp_dir_out, f'{year}.html')\n",
+    "  fp_out_csv = join(fp_dir_out, f'{year}.csv')\n",
+    "  participants = parse_year(fp_in_html, year)\n",
+    "  participants_all += participants\n",
+    "  df = pd.DataFrame.from_dict(participants)\n",
+    "  df.to_csv(fp_out_csv, index=False)\n",
+    "  print(f'Wrote: {fp_out_csv} with {len(participants)} items')\n",
+    "\n",
+    "# write total list\n",
+    "\n",
+    "df = pd.DataFrame.from_dict(participants_all)\n",
+    "df.to_csv(fp_out_all_csv, index=False)\n",
+    "print(f'Wrote: {fp_out_all_csv} with {len(participants_all)} items')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 94,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 95,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "megapixels",
+   "language": "python",
+   "name": "megapixels"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}