From efadb39172eb36b4ffebeaa5169219c7954a263b Mon Sep 17 00:00:00 2001
From: Jules Laplace <julescarbon@gmail.com>
Date: Sun, 10 Feb 2019 16:38:53 +0100
Subject: parallelize fetch pdf

---
 scraper/util.py | 7 +++++++
 1 file changed, 7 insertions(+)

(limited to 'scraper/util.py')

diff --git a/scraper/util.py b/scraper/util.py
index 7b55afae..d3f4e751 100644
--- a/scraper/util.py
+++ b/scraper/util.py
@@ -4,6 +4,7 @@ import csv
 import string
 import codecs
 import gspread
+from multiprocessing import Pool
 import simplejson as json
 from oauth2client.service_account import ServiceAccountCredentials
 
@@ -302,6 +303,12 @@ class AddressBook (object):
       }
     return None
 
+def parallelize(func, rows):
+  print("Fetching {} items".format(len(rows)))
+  chunksize = 3
+  with Pool(processes=len(os.sched_getaffinity(0))) as pool:
+    pool.starmap(func, rows, chunksize)
+
 def fetch_paper(s2, paper_id):
   os.makedirs('./datasets/s2/papers/{}/{}'.format(paper_id[0:2], paper_id), exist_ok=True)
   paper_fn = './datasets/s2/papers/{}/{}/paper.json'.format(paper_id[0:2], paper_id)
-- 
cgit v1.2.3-70-g09d2