check if files exist before putting them in the processing pipelineHEADmaster

1 files changed, 9 insertions, 1 deletions

diff --git a/cli/app/utils/cortex_utils.py b/cli/app/utils/cortex_utils.py
index 7974502..e40edbc 100644
--- a/cli/app/utils/cortex_utils.py
+++ b/cli/app/utils/cortex_utils.py
@@ -12,9 +12,11 @@ def api_url(path):
   return "{}/api/{}/".format(app_cfg.API_REMOTE, path)
 
 def cortex_folder(opt_folder_id):
+  """Retrieve a folder (list of files) from the cortex"""
   return fetch_json(os.path.join(api_url('folder'), str(opt_folder_id) + "/"))
 
 def results_folder(name="results", module="biggan"):
+  """Retrieve the results folder from the cortex (should have been auto-generated)"""
   res = fetch_json(api_url('folder'), name=name, module=module)
   if len(res):
     return res[0]
@@ -25,14 +27,20 @@ def download_cortex_files(opt_folder_id):
   rows = fetch_json(api_url('file'), folder_id=opt_folder_id)
   fp_out_dir = join(app_cfg.DIR_INPUTS, "cortex", str(opt_folder_id))
   os.makedirs(fp_out_dir, exist_ok=True)
+  valid_rows = []
   for row in rows:
+    # first, fetch any new images that haven't been processed
     if row['generated'] == 0 and row['processed'] != 1:
       fn, ext = os.path.splitext(row['name'])
       fp_out_image = join(fp_out_dir, row['name'])
       row['path'] = fp_out_image
       if not os.path.exists(fp_out_image):
         fetch_file(row['url'], fp_out_image)
-  return rows
+    # then check if it actually downloaded the files
+    fp_out_image = join(fp_out_dir, row['name'])
+    if os.path.exists(fp_out_image):
+      valid_rows.append(row)
+  return valid_rows
 
 def find_unprocessed_files(files, reprocess=False):
   """Find files that haven't been processed yet.