diff options
Diffstat (limited to 'cli/app/utils')
| -rw-r--r-- | cli/app/utils/cortex_utils.py | 10 |
1 files changed, 9 insertions, 1 deletions
diff --git a/cli/app/utils/cortex_utils.py b/cli/app/utils/cortex_utils.py index 7974502..e40edbc 100644 --- a/cli/app/utils/cortex_utils.py +++ b/cli/app/utils/cortex_utils.py @@ -12,9 +12,11 @@ def api_url(path): return "{}/api/{}/".format(app_cfg.API_REMOTE, path) def cortex_folder(opt_folder_id): + """Retrieve a folder (list of files) from the cortex""" return fetch_json(os.path.join(api_url('folder'), str(opt_folder_id) + "/")) def results_folder(name="results", module="biggan"): + """Retrieve the results folder from the cortex (should have been auto-generated)""" res = fetch_json(api_url('folder'), name=name, module=module) if len(res): return res[0] @@ -25,14 +27,20 @@ def download_cortex_files(opt_folder_id): rows = fetch_json(api_url('file'), folder_id=opt_folder_id) fp_out_dir = join(app_cfg.DIR_INPUTS, "cortex", str(opt_folder_id)) os.makedirs(fp_out_dir, exist_ok=True) + valid_rows = [] for row in rows: + # first, fetch any new images that haven't been processed if row['generated'] == 0 and row['processed'] != 1: fn, ext = os.path.splitext(row['name']) fp_out_image = join(fp_out_dir, row['name']) row['path'] = fp_out_image if not os.path.exists(fp_out_image): fetch_file(row['url'], fp_out_image) - return rows + # then check if it actually downloaded the files + fp_out_image = join(fp_out_dir, row['name']) + if os.path.exists(fp_out_image): + valid_rows.append(row) + return valid_rows def find_unprocessed_files(files, reprocess=False): """Find files that haven't been processed yet. |