{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Get Embassies on Flickr" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "ename": "ModuleNotFoundError", "evalue": "No module named 'urllib2'", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mbs4\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mBeautifulSoup\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0murllib\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0murllib2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 14\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mre\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'urllib2'" ] } ], "source": [ "%reload_ext autoreload\n", "%autoreload 2\n", "\n", "import os\n", "from os.path import join\n", "from pathlib import Path\n", "from glob import glob\n", "import json\n", "from pprint import pprint\n", "\n", "from bs4 import BeautifulSoup\n", "import urllib\n", "import urllib2\n", "import re\n", "import numpy as np\n", "import pandas as pd\n", "from PIL import Image, ImageDraw\n", "%matplotlib inline\n", "import matplotlib.pyplot as plt\n", "from tqdm import tqdm_notebook as tqdm" ] }, { "cell_type": "code", "execution_count": 74, "metadata": {}, "outputs": [], "source": [ "url = \"http://duckduckgo.com/html/?q=\"\n", "query = 'flickr.com+us+embassy'\n", "fp_html = '/data_store/datasets/msc/us_embassy.html'\n", "with open(fp_html,'r') as fp:\n", " html = fp.readlines()\n", "html = html[0]" ] }, { "cell_type": "code", "execution_count": 76, "metadata": {}, "outputs": [], "source": [ "soup = BeautifulSoup(html, \"html.parser\")" ] }, { "cell_type": "code", "execution_count": 77, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Found 169 results\n", "US Embassy | Flickr https://www.flickr.com/photos/us_embassy_newzealand/\n", "Flickr: US Embassy New Zealand https://www.flickr.com/people/us_embassy_newzealand/\n", "U.S. Embassy London | Flickr https://www.flickr.com/photos/usembassylondon/\n", "United States Embassy Kuala Lumpur | Flickr https://www.flickr.com/photos/usembassykl/\n", "US Embassy Tegucigalpa | Flickr https://www.flickr.com/photos/usembassyteg/\n", "US Embassy Guatemala | Flickr https://www.flickr.com/photos/usembassyguatemala\n", "US Embassy South Africa | Flickr https://www.flickr.com/photos/USEmbassySA\n", "US Embassy Seoul | Flickr https://www.flickr.com/photos/usembassyseoul/\n", "Flickr: US Embassy Athens' Photostream https://www.flickr.com/photos/40169011@N07/\n", "Flickr: US Embassy New Zealand's Photostream https://www.flickr.com/photos/46907600@N02\n", "Flickr: U.S. Embassy Nairobi's Photostream https://www.flickr.com/photos/us_embassy_nairobi/\n", "US Embassy Sweden | Flickr https://www.flickr.com/photos/usembsweden\n", "Flickr: U.S. Embassy Bern, Switzerland's Photostream https://www.flickr.com/photos/usembassybern/page3\n", "US Embassy Panama | Flickr https://www.flickr.com/photos/usembassypanama\n", "U.S. Embassy Romania | Flickr https://www.flickr.com/photos/usembassyromania/\n", "U.S. Embassy Bern, Switzerland | Flickr https://www.flickr.com/photos/usembassybern/\n", "U.S. Embassy in the Philippines | Flickr https://www.flickr.com/photos/usembassymanila\n", "US Embassy Lisbon Portugal | Flickr https://www.flickr.com/photos/usembassylisbon/\n", "U.S. Embassy Namibia | Flickr https://www.flickr.com/photos/usembassynamibia/\n", "Flickr: U.S. Embassy Lilongwe's Photostream https://www.flickr.com/photos/usembassylilongwe/\n", "usembassykyiv - Find your inspiration. | Flickr https://www.flickr.com/people/usembassykyiv/\n", "Flag Day | To celebrate Flag Day, the U.S. Embassy's Public ... https://www.flickr.com/photos/usembassyromania/7401175050/\n", "U.S. Embassy Yaounde | Flickr https://www.flickr.com/photos/usembyaounde/\n", "New London Embassy Construction | Flickr https://www.flickr.com/photos/usembassylondon/albums/72157635166293639/\n", "U.S. Embassy, Jakarta | Flickr https://www.flickr.com/photos/usembassyjakarta/\n", "U.S. Embassy Valletta, Malta's albums | Flickr https://www.flickr.com/photos/usembmalta/albums\n", "Embajada de los Estados Unidos en Bogotá | Flickr https://www.flickr.com/photos/usembassybogota\n", "Michael A. McCarthy | Flickr https://www.flickr.com/photos/usembassysa/albums/72157683619571002/\n", "U.S. Embassy Nigeria | Flickr https://www.flickr.com/photos/usembassynigeria\n", "U.S. Embassy Ghana | Flickr https://www.flickr.com/photos/usembghana/\n", "U.S. Embassy Dhaka | Flickr https://www.flickr.com/photos/usembassydhaka\n", "US Embassy Canada | Flickr https://www.flickr.com/photos/us_mission_canada/\n", "Flickr: U.S. Embassy Tokyo https://www.flickr.com/people/usembassytokyo/\n", "Collection: Ottawa - U.S. Embassy - Flickr https://www.flickr.com/photos/us_mission_canada/collections/72157622101856611/\n", "U.S. Embassy Bamako | Flickr https://www.flickr.com/photos/usembassymali\n", "Flickr: US Embassy Honduras https://www.flickr.com/people/usembassyteg/\n", "US Embassy Guatemala's albums | Flickr https://www.flickr.com/photos/usembassyguatemala/albums\n", "usembassy photos on Flickr | Flickr https://www.flickr.com/photos/tags/usembassy\n", "U.S. Embassy, Singapore | Flickr https://www.flickr.com/photos/singaporeusembassy/\n", "US Embassy Panama's albums | Flickr https://www.flickr.com/photos/usembassypanama/albums/\n", "U.S. Embassy Brasilia | Flickr https://www.flickr.com/photos/embaixadaeua-brasil/\n", "US Embassy San Salvador | Flickr https://www.flickr.com/photos/40236643@N04/\n", "US Embassy Kabul Afghanistan | Flickr https://www.flickr.com/photos/kabulpublicdiplomacy\n", "Flickr: U.S. Embassy Montevideo https://www.flickr.com/people/usembassy_montevideo/\n", "5 - The U.S. Civil Rights Movement | Members of the ... - Flickr https://www.flickr.com/photos/usembassythehague/8443765507\n", "US ARMY 1969-1970 South photos on Flickr | Flickr https://www.flickr.com/photos/tags/US%20ARMY%201969-1970%20South/\n", "Flickr: usembassylondon https://www.flickr.com/people/usembassylondon/\n", "Visita del Contraalmirante Sean Buck - an album on Flickr https://www.flickr.com/photos/usembassy_montevideo/albums/72157679561406348\n", "US Embassy Madrid | Flickr https://www.flickr.com/photos/usembassymadrid\n", "Flickr: U.S. Embassy in the Philippines https://www.flickr.com/people/42751184@N06\n", "US Embassy's albums | Flickr https://www.flickr.com/photos/us_embassy_newzealand/albums/\n", "U.S. Embassy Canberra | Flickr https://www.flickr.com/photos/usembassycanberra/\n", "PAS China | Flickr https://www.flickr.com/photos/usembassybeijing\n", "U.S. Embassy Kyiv Ukraine | Flickr https://www.flickr.com/photos/usembassykyiv/\n", "Flickr: US Embassy Canada https://www.flickr.com/people/us_mission_canada/\n", "Secretary of State Hillary Clinton in New Zealand - the se ... https://www.flickr.com/photos/us_embassy_newzealand/5156414057\n", "US Embassy Kuwait | Flickr https://www.flickr.com/photos/usembassyq8\n", "US Embassy Lisbon Portugal's albums | Flickr https://www.flickr.com/photos/usembassylisbon/albums\n", "Flickr: US Embassy Guatemala https://www.flickr.com/people/usembassyguatemala/\n", "Flickr: U.S. Embassy Panama https://www.flickr.com/people/usembassypanama/\n", "U.S. Embassy France | Flickr https://www.flickr.com/photos/usembassyfrance/\n", "US Embassy Dakar | Flickr https://www.flickr.com/photos/usembassydakar/\n", "U.S. Embassy Tashkent | Flickr https://www.flickr.com/photos/us-embassy-tashkent/\n", "USEmbassy Riga | Flickr https://www.flickr.com/photos/usembassyriga/\n", "US Embassy Santiago, Chile's albums | Flickr https://www.flickr.com/photos/embajadaeeuu-chile/albums/\n", "Flickr: Embassy of Italy in the US https://www.flickr.com/people/italianembassy/\n", "Cameron Gawley from Buzzshift, in New Zealand for The Proj ... https://www.flickr.com/photos/us_embassy_newzealand/14138238773\n", "Flickr: USEmbassyTbilisi https://www.flickr.com/people/usembassytbilisi/\n", "US Embassy South Africa's albums | Flickr https://www.flickr.com/photos/usembassysa/albums\n", "Flickr: U.S. Embassy Bern, Switzerland https://www.flickr.com/people/usembassybern/\n", "Flickr: Embassy of Argentina in the US' Photostream https://www.flickr.com/photos/ARGinUSA\n", "Flickr: U.S. Embassy The Hague https://www.flickr.com/people/usembassythehague/\n", "6 - The U.S. Civil Rights Movement | \"Freedom Riders\" hang a ... https://www.flickr.com/photos/usembassythehague/8443765523\n", "2018-11-05 EducationUSA- San Mateo College and UC Berkly https://www.flickr.com/photos/usembassyphnompenh/albums/72157704091251915/\n", "U.S. Embassy Vienna | Flickr https://www.flickr.com/photos/usembassy_vienna/\n", "US Embassy Colombo | Flickr https://www.flickr.com/photos/usembassycolombo\n", "U.S. Embassy Qatar | Flickr https://www.flickr.com/photos/usembassyqatar/\n", "us embassy | Flickr https://www.flickr.com/photos/usembassy/\n", "Nepal Trade Preference Program - 2017 | Flickr https://www.flickr.com/photos/usembassykathmandu/albums/72157679864242964/\n", "Flickr: U.S. Embassy New Delhi https://www.flickr.com/people/usembassynewdelhi/\n", "Flickr: U.S. Embassy Dushanbe https://www.flickr.com/people/usembassydushanbe/\n", "United States Embassy Kuala Lumpur's albums | Flickr https://www.flickr.com/photos/usembassykl/albums/\n", "Ambassador Glass Presentation of Credentials - flickr.com https://www.flickr.com/photos/usembassylisbon/36915173035/in/album-72157685920310283/\n", "U.S. Embassy, Jakarta | Flickr https://www.flickr.com/photos/39809323@N03/\n", "US Embassy, Jul 1965, Seoul | US Embassy, Ulchi-ro. Flag at ... https://www.flickr.com/photos/88857036@N00/4182215366\n", "U.S. Embassy New Delhi's albums | Flickr https://www.flickr.com/photos/usembassynewdelhi/albums/\n", "Roosevelt reads from a Dutch family Bible before his ... https://www.flickr.com/photos/usembassythehague/8369526559\n", "Collection: Montreal - U.S. Consulate General https://www.flickr.com/photos/us_mission_canada/collections/72157622226501154/\n", "shubra photos on Flickr | Flickr https://www.flickr.com/photos/tags/shubra/\n", "US Embassy Canada | Flickr https://www.flickr.com/photos/38144472@N04\n", "Flickr: U.S. Embassy Djibouti https://www.flickr.com/people/usembassydjibouti/\n", "President Obama Celebrating Birthdays - flickr.com https://www.flickr.com/photos/usembassynewdelhi/6000570229/\n", "U.S. Embassy Celebrates the American Road Trip on ... https://www.flickr.com/photos/usembassythehague/albums/72157698243008854\n", "Education USA Press Photos | Flickr https://www.flickr.com/photos/usembassysa/albums/72157684493034910/\n", "US Embassy Suva | Flickr https://www.flickr.com/photos/167418767@N06\n", "Flickr: U.S. Embassy Romania https://www.flickr.com/people/usembassyromania/\n", "U.S. Embassy Oslo Historical Photos | Flickr https://www.flickr.com/photos/usembassyoslo/albums/72157655819966261/\n", "United States Embassy South Sudan | Flickr https://www.flickr.com/photos/usembassysouthsudan\n", "usembassybelgium | Flickr https://www.flickr.com/photos/usembassybelgium/\n", "U.S. Embassy Valletta, Malta | Flickr https://www.flickr.com/photos/usembmalta\n", "U.S. Embassy Moldova | Flickr https://www.flickr.com/photos/usembassymoldova\n", "4 - The U.S. Civil Rights Movement | September 4, 1957, Eliz ... https://www.flickr.com/photos/usembassythehague/8443765489\n", "U.S. Embassy Dublin's albums | Flickr https://www.flickr.com/photos/usembassydub/albums/\n", "US Consul General Sherry Zalika Sykes - Flickr https://www.flickr.com/photos/usembassysa/36571316593\n", "United States Embassy Zambia, Lusaka | Flickr https://www.flickr.com/photos/usembassyzambia\n", "U.S. Response to the Ethiopian Drought | Flickr https://www.flickr.com/photos/usembassyaddisababa/albums/72157661696341960/\n", "Visit of U.S.S. Sampson to Auckland, November 22, 2016 - an ... https://www.flickr.com/photos/us_embassy_newzealand/albums/72157672962462663\n", "U.S. Embassy Tunis | Flickr https://www.flickr.com/photos/usembassytunis/\n", "US Embassy Kabul logo | US Embassy Kabul Afghanistan | Flickr https://www.flickr.com/photos/kabulpublicdiplomacy/3244151780\n", "Ambassador Hoekstra visits Fryslân - an album on Flickr https://www.flickr.com/photos/usembassythehague/albums/72157693808118171\n", "2018-11-07 Career Development Club- Overview on Project ... https://www.flickr.com/photos/usembassyphnompenh/albums/72157704552786665/\n", "Ambassador Rubin Visits U.S. Company Cargill Facilities in ... https://www.flickr.com/photos/usembassysofia/albums/72157690755210113\n", "LIMA Exhibition 2019 | Flickr https://www.flickr.com/photos/usembassykl/albums/72157679554062678\n", "US Embassy Santiago, Chile | Flickr https://www.flickr.com/photos/embajadaeeuu-chile/\n", "11 - Civil Rights Movement - Flickr https://www.flickr.com/photos/usembassythehague/8444909206/\n", "U.S. Embasy in Tbilisi | Flickr https://www.flickr.com/photos/usembassytbilisi\n", "US Embassy - flickr.com https://www.flickr.com/photos/13476480@N07/4049465204\n", "Late February - The Year 12 History class at Pakuranga ... https://www.flickr.com/photos/us_embassy_newzealand/4397231272/\n", "U.S. Embassy Ankara,Turkey | ABD Büyükelçiliği ... - Flickr https://www.flickr.com/photos/usembassyankara/albums\n", "Dinner for Karen Diver, hosted by Ambassador Gilbert | Flickr https://www.flickr.com/photos/us_embassy_newzealand/30382649570/\n", "Independence Day 2015 | Flickr https://www.flickr.com/photos/usembassylisbon/albums/72157653326238334/\n", "US Embassy Panama | Flickr https://www.flickr.com/photos/52474456@N04/\n", "U.S. Embassy Khartoum | Flickr https://www.flickr.com/photos/usembassykhartoum/\n", "New American embassy building site at Huseby | Flickr https://www.flickr.com/photos/usembassyoslo/albums/72157655830040705/\n", "Proyecto ECHO (CDC/PEPFAR) - an album on Flickr https://www.flickr.com/photos/usembassyguatemala/sets/72157688137447250\n", "Flickr: US Embassy in Costa Rica https://www.flickr.com/people/usembassysjo/\n", "2018-12-14 Film Screening of \"Alice's Ordinary People The ... https://www.flickr.com/photos/usembassyphnompenh/albums/72157704618742275/\n", "Flickr: USEmbassyPhnomPenh https://www.flickr.com/people/usembassyphnompenh/\n", "Secretary of Defence James Mattis' Visit to the United ... https://www.flickr.com/photos/usembassylondon/albums/72157681958372486/\n", "United States - Flickr https://www.flickr.com/groups/64566829@N00/discuss/72157642028711915/\n", "Preserving Liliesleaf Farm's Rare Documents - an album on Flickr https://www.flickr.com/photos/usembassysa/sets/72157675103512465/\n", "Art in the U.S. Embassy Ottawa | Flickr https://www.flickr.com/photos/us_mission_canada/albums/72157625319788766/\n", "Flickr: US Embassy San Salvador https://www.flickr.com/people/40236643@N04/\n", "US Embassy | Matt Brown | Flickr https://www.flickr.com/photos/londonmatt/11606999005\n", "The Ambassadors Christmas Tree Event | Flickr https://www.flickr.com/photos/usembsweden/albums/72157666927606479\n", "100th Anniversary U.S. Marines in Ponta Delgada | Flickr https://www.flickr.com/photos/usembassylisbon/sets/72157682736277172/\n", "U.S. Embassy, Jakarta's albums | Flickr https://www.flickr.com/photos/usembassyjakarta/albums/\n", "Ambassador Scott Brown and Gail Brown on their second visit ... https://www.flickr.com/photos/us_embassy_newzealand/albums/72157692928434925\n", "Flickr: US Embassy Baku https://www.flickr.com/people/usembassybaku/\n", "The Flags | Photo by Pascal Cornioley | U.S. Embassy Bern ... https://www.flickr.com/photos/usembassybern/7507448592\n", "Vietnam Saigon Bombing 1965 | Injured Vietnamese ... - Flickr https://www.flickr.com/photos/thienmai/6547036861/\n", "US Embassy Venezuela | Flickr https://www.flickr.com/photos/usembassyve\n", "Vital Voices en Quiché (PAS) - an album on Flickr https://www.flickr.com/photos/usembassyguatemala/albums/72157687937588055/\n", "Dangdut Cowboys Hadir di Indonesia Meriahkan Peringatan 70 ... https://www.flickr.com/photos/usembassyjakarta/albums/72157677565530337\n", "Flickr: U.S. Embassy Dublin https://www.flickr.com/people/usembassydub/\n", "PA US Embassy Maputo | Flickr https://www.flickr.com/photos/usembassymaputo/\n", "Jim Adams - NASA | Flickr https://www.flickr.com/photos/usembassysa/albums/72157677976059474/\n", "AmeriCarna Reception, 22 February 2011 - Flickr https://www.flickr.com/photos/us_embassy_newzealand/5469177289/\n", "Middle School Students at Knightsbridge Schools Internatio ... https://www.flickr.com/photos/usembassypanama/14204669983\n", "Rona Rathod, U.S. Embassy Rule of Law adviser, talks with ... https://www.flickr.com/photos/kabulpublicdiplomacy/5491661477\n", "U.S. Embassy Djibouti's groups | Flickr https://www.flickr.com/people/usembassydjibouti/groups/\n", "Nieuwjaarsduik 2017 | Flickr https://www.flickr.com/photos/usembassythehague/albums/72157674988873153/\n", "United States Special Representative for Ukraine Negotiati ... https://www.flickr.com/photos/usembassykyiv/35325890653\n", "Embajada de los Estados Unidos en Uruguay | Flickr https://www.flickr.com/photos/usembassy_montevideo/\n", "La embajadora Powers junto a mujeres de la comunidad Ember ... https://www.flickr.com/photos/usembassypanama/5731319608\n", "US Embassy Seoul's groups | Flickr https://www.flickr.com/people/usembassyseoul/groups/\n", "Holiday Party de la Embajada de EE.UU. | Flickr https://www.flickr.com/photos/usembassypanama/albums/72157703158663661\n", "PA US Embassy Maputo's albums | Flickr https://www.flickr.com/photos/usembassymaputo/albums/\n", "Veterans Day 2018 | Flickr https://www.flickr.com/photos/usembassyluxembourg/albums/72157703436301924\n", "US Embassy Canada's albums | Flickr https://www.flickr.com/photos/us_mission_canada/albums/\n", "Flickr: U.S. Embassy Jerusalem https://www.flickr.com/people/46886434@N04/\n", "Juego de Fútbol Estados Unidos vs Panamá | Flickr https://www.flickr.com/photos/usembassypanama/albums/72157680270741830/\n", "Abidjan evacuation, April 2011 | U.S. Embassy staff ... - Flickr https://www.flickr.com/photos/usarmyafrica/5693157628\n", "Super Chef Panama 2018 | Flickr https://www.flickr.com/photos/usembassypanama/albums/72157694410354945\n", "New United States Embassy in South VN (rear view) - Flickr https://www.flickr.com/photos/13476480@N07/3757202154\n", "Flickr: US Department of State https://www.flickr.com/people/iip-photo-archive/\n", "Seal of the Consulate General | The Great Seal of the United ... https://www.flickr.com/photos/us_mission_canada/4770405952/\n", "Senator Bob Corker (R-TN) and delegation visit the U.S ... https://www.flickr.com/photos/kabulpublicdiplomacy/albums/72157626093355080/\n", "Flickr: Embajada de Estados Unidos en Perú's Photostream https://www.flickr.com/photos/usembassyperu\n" ] } ], "source": [ "topics = soup.find_all('div', attrs={'class': 'results_links_deep'})\n", "print(f'Found {len(topics)} results')\n", "for topic in topics:\n", " title = topic.find('h2', {'class': 'result__title'})\n", " ahref = title.find('a')\n", " url = ahref.get('href')\n", " if 'flickr.com' in url:\n", " print(ahref.text, ahref.get('href'))\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print(results[0].text)" ] } ], "metadata": { "kernelspec": { "display_name": "megapixels", "language": "python", "name": "megapixels" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.7" } }, "nbformat": 4, "nbformat_minor": 2 }