1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
|
import os
from os.path import join
import requests
import urllib3
import mimetypes
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
from app.settings import app_cfg
def api_url(path):
"""Generate a base API path"""
return "https://lens.neural.garden/api/{}/".format(path)
def cortex_folder(opt_folder_id):
return fetch_json(os.path.join(api_url('folder'), str(opt_folder_id) + "/"))
def results_folder(name="results", module="biggan"):
res = fetch_json(api_url('folder'), name=name, module=module)
if len(res):
return res[0]
raise Exception("No results folder!")
def download_cortex_files(opt_folder_id):
"""Fetch all new, non-generated files in a Cortex folder"""
rows = fetch_json(api_url('file'), folder_id=opt_folder_id)
fp_out_dir = join(app_cfg.DIR_INPUTS, "cortex", str(opt_folder_id))
os.makedirs(fp_out_dir, exist_ok=True)
for row in rows:
if row['generated'] == 0 and row['processed'] != 1:
fn, ext = os.path.splitext(row['name'])
fp_out_image = join(fp_out_dir, row['name'])
row['path'] = fp_out_image
if not os.path.exists(fp_out_image):
fetch_file(row['url'], fp_out_image)
return rows
def find_unprocessed_files(files, reprocess=False):
"""Find files that haven't been processed yet.
This is implied if no matching generated file is found.
"""
datasets = {}
unprocessed_files = []
for file in files:
if file['generated'] == 1 and file['datatype'] == 'image':
fn, ext = os.path.splitext(file['name'])
dataset = fn.split('-')[0]
datasets[dataset] = file['id']
for file in files:
if file['generated'] == 0 and file['processed'] == 0 and file['datatype'] == 'image':
fn, ext = os.path.splitext(file['name'])
dataset = fn.split('-')[0]
if dataset not in datasets or reprocess == True:
unprocessed_files.append(file)
return unprocessed_files
def fetch_json(url, **kwargs):
"""HTTP GET some JSON"""
resp = requests.get(url, params=kwargs, verify=False, timeout=10)
return None if resp.status_code != 200 else resp.json()
def fetch_file(url, fn, **kwargs):
"""HTTP GET a binary file and write it to disk"""
print("Fetch {} => {}".format(url, fn))
try:
resp = requests.get(url, params=kwargs, verify=False, timeout=10)
if resp.status_code != 200:
return None
except:
return None
size = 0
with open(fn, 'wb') as f:
for chunk in resp.iter_content(chunk_size=1024):
if chunk:
size += len(chunk)
f.write(chunk)
return size
def upload_fp_to_cortex(opt_folder_id, fp, generated='true', module='biggan', activity='invert', datatype='image'):
"""Upload an open file/BytesIO object"""
files = {
'file': fp
}
data = {
'folder_id': opt_folder_id,
'generated': generated,
'module': module,
'activity': activity,
'datatype': datatype,
}
url = os.path.join(api_url('folder'), str(opt_folder_id), 'upload/')
r = requests.post(url, files=files, data=data)
return None if r.status_code != 200 else r.json()
def upload_bytes_to_cortex(opt_folder_id, fn, fp, mimetype, **kwargs):
"""Upload a BytesIO object"""
return upload_fp_to_cortex(opt_folder_id, (fn, fp.getvalue(), mimetype,), **kwargs)
def upload_file_to_cortex(opt_folder_id, fn, **kwargs):
"""Upload a file from disk"""
base_fn = os.path.basename(fn)
mimetype_tup = mimetypes.guess_type(base_fn)
mime_name = mimetype_tup[1]
if '/' in mime_name:
mime_name = mime_name.split('/')[0]
mimetype = "{}/{}".format(mimetype_tup[0], mime_name)
with open(fn, 'rb') as fp:
return upload_fp_to_cortex(opt_folder_id, (base_fn, fp, mimetype,), **kwargs)
|
