check if files exist before putting them in the processing pipelineHEADmaster

2 files changed, 10 insertions, 2 deletions

diff --git a/cli/app/search/search_class.py b/cli/app/search/search_class.py
index b0c95bb..901ee0c 100644
--- a/cli/app/search/search_class.py
+++ b/cli/app/search/search_class.py
@@ -71,7 +71,7 @@ def find_nearest_vector_for_images(paths, opt_dims, opt_steps, opt_video, opt_ta
     out_fns[index] = os.path.basename(path)
     fp_frames = find_nearest_vector(sess, generator, path, opt_dims, out_images, out_labels, out_latent, opt_steps, index, opt_tag,
       opt_stochastic_clipping, opt_label_clipping, opt_use_feature_detector, opt_feature_layers, opt_snapshot_interval, opt_clip_interval, opt_video)
-    if opt_video:
+    if opt_video and fp_frames is not None:
       export_video(fp_frames)
   sess.close()
 
diff --git a/cli/app/utils/cortex_utils.py b/cli/app/utils/cortex_utils.py
index 7974502..e40edbc 100644
--- a/cli/app/utils/cortex_utils.py
+++ b/cli/app/utils/cortex_utils.py
@@ -12,9 +12,11 @@ def api_url(path):
   return "{}/api/{}/".format(app_cfg.API_REMOTE, path)
 
 def cortex_folder(opt_folder_id):
+  """Retrieve a folder (list of files) from the cortex"""
   return fetch_json(os.path.join(api_url('folder'), str(opt_folder_id) + "/"))
 
 def results_folder(name="results", module="biggan"):
+  """Retrieve the results folder from the cortex (should have been auto-generated)"""
   res = fetch_json(api_url('folder'), name=name, module=module)
   if len(res):
     return res[0]
@@ -25,14 +27,20 @@ def download_cortex_files(opt_folder_id):
   rows = fetch_json(api_url('file'), folder_id=opt_folder_id)
   fp_out_dir = join(app_cfg.DIR_INPUTS, "cortex", str(opt_folder_id))
   os.makedirs(fp_out_dir, exist_ok=True)
+  valid_rows = []
   for row in rows:
+    # first, fetch any new images that haven't been processed
     if row['generated'] == 0 and row['processed'] != 1:
       fn, ext = os.path.splitext(row['name'])
       fp_out_image = join(fp_out_dir, row['name'])
       row['path'] = fp_out_image
       if not os.path.exists(fp_out_image):
         fetch_file(row['url'], fp_out_image)
-  return rows
+    # then check if it actually downloaded the files
+    fp_out_image = join(fp_out_dir, row['name'])
+    if os.path.exists(fp_out_image):
+      valid_rows.append(row)
+  return valid_rows
 
 def find_unprocessed_files(files, reprocess=False):
   """Find files that haven't been processed yet.