pdf downloader

1 files changed, 20 insertions, 4 deletions

diff --git a/s2.py b/s2.py
index c9de75e6..696b1a45 100644
--- a/s2.py
+++ b/s2.py
@@ -116,17 +116,33 @@ class SemanticScholarAPI(object):
     PAPER_ENDPOINT = "{}/{}".format(BASE_URL, "paper")
     SEARCH_ENDPOINT = "https://www.semanticscholar.org/api/1/search"
     RAW_PAPER_ENDPOINT = "https://www.semanticscholar.org/api/1/paper"
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36',
+    }
+
+    @staticmethod
+    def fetch_file(url, fn, **kwargs):
+        resp = requests.get(url, params=kwargs, headers=SemanticScholarAPI.headers)
+        if resp.status_code != 200:
+            return None
+        size = 0
+        with open(fn, 'wb') as f:
+            for chunk in resp.iter_content(chunk_size=1024):
+                if chunk:
+                    size += len(chunk)
+                    f.write(chunk)
+        return size
 
     @staticmethod
     def paper(paper_id, **kwargs):
         url = "{}/{}".format(SemanticScholarAPI.PAPER_ENDPOINT, paper_id)
-        resp = requests.get(url, params=kwargs)
+        resp = requests.get(url, params=kwargs, headers=SemanticScholarAPI.headers)
         return None if resp.status_code != 200 else resp.json() # Paper(**resp.json())
 
     @staticmethod
     def author(author_id, **kwargs):
         url = "{}/{}".format(SemanticScholarAPI.AUTHOR_ENDPOINT, author_id)
-        resp = requests.get(url, params=kwargs)
+        resp = requests.get(url, params=kwargs, headers=SemanticScholarAPI.headers)
         return None if resp.status_code != 200 else resp.json() # Author(**resp.json())
 
     @staticmethod
@@ -136,7 +152,7 @@ class SemanticScholarAPI(object):
     @staticmethod
     def raw_paper(paper_id, **kwargs):
         url = "{}/{}".format(SemanticScholarAPI.RAW_PAPER_ENDPOINT, paper_id)
-        resp = requests.get(url, params=kwargs)
+        resp = requests.get(url, params=kwargs, headers=SemanticScholarAPI.headers)
         return None if resp.status_code != 200 else resp.json() # Paper(**resp.json())
 
     @staticmethod
@@ -153,6 +169,6 @@ class SemanticScholarAPI(object):
             'sort': "relevance",
             'venues': [],
             'yearFilter': None,
-        })
+        }, headers=SemanticScholarAPI.headers)
         # print(resp.status_code)
         return None if resp.status_code != 200 else resp.json()