From 574562a39fb9b08d3bb6e4383e9b04aa3029bad6 Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Fri, 9 Oct 2020 13:58:07 +0200 Subject: check if files exist before putting them in the processing pipeline --- cli/app/search/search_class.py | 2 +- cli/app/utils/cortex_utils.py | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) (limited to 'cli') diff --git a/cli/app/search/search_class.py b/cli/app/search/search_class.py index b0c95bb..901ee0c 100644 --- a/cli/app/search/search_class.py +++ b/cli/app/search/search_class.py @@ -71,7 +71,7 @@ def find_nearest_vector_for_images(paths, opt_dims, opt_steps, opt_video, opt_ta out_fns[index] = os.path.basename(path) fp_frames = find_nearest_vector(sess, generator, path, opt_dims, out_images, out_labels, out_latent, opt_steps, index, opt_tag, opt_stochastic_clipping, opt_label_clipping, opt_use_feature_detector, opt_feature_layers, opt_snapshot_interval, opt_clip_interval, opt_video) - if opt_video: + if opt_video and fp_frames is not None: export_video(fp_frames) sess.close() diff --git a/cli/app/utils/cortex_utils.py b/cli/app/utils/cortex_utils.py index 7974502..e40edbc 100644 --- a/cli/app/utils/cortex_utils.py +++ b/cli/app/utils/cortex_utils.py @@ -12,9 +12,11 @@ def api_url(path): return "{}/api/{}/".format(app_cfg.API_REMOTE, path) def cortex_folder(opt_folder_id): + """Retrieve a folder (list of files) from the cortex""" return fetch_json(os.path.join(api_url('folder'), str(opt_folder_id) + "/")) def results_folder(name="results", module="biggan"): + """Retrieve the results folder from the cortex (should have been auto-generated)""" res = fetch_json(api_url('folder'), name=name, module=module) if len(res): return res[0] @@ -25,14 +27,20 @@ def download_cortex_files(opt_folder_id): rows = fetch_json(api_url('file'), folder_id=opt_folder_id) fp_out_dir = join(app_cfg.DIR_INPUTS, "cortex", str(opt_folder_id)) os.makedirs(fp_out_dir, exist_ok=True) + valid_rows = [] for row in rows: + # first, fetch any new images that haven't been processed if row['generated'] == 0 and row['processed'] != 1: fn, ext = os.path.splitext(row['name']) fp_out_image = join(fp_out_dir, row['name']) row['path'] = fp_out_image if not os.path.exists(fp_out_image): fetch_file(row['url'], fp_out_image) - return rows + # then check if it actually downloaded the files + fp_out_image = join(fp_out_dir, row['name']) + if os.path.exists(fp_out_image): + valid_rows.append(row) + return valid_rows def find_unprocessed_files(files, reprocess=False): """Find files that haven't been processed yet. -- cgit v1.2.3-70-g09d2