import click import os from app.utils.cortex_utils import fetch_cortex_folder, find_unprocessed_files from app.search.search_class import find_nearest_vector_for_images from app.search.search_dense import find_dense_embedding_for_images from app.search.json import params_dense_dict @click.command('') @click.option('-f', '--folder_id', 'opt_folder_id', type=int, help='Folder ID to process') @click.option('-ls', '--latent_steps', 'opt_latent_steps', default=2000, type=int, help='Number of optimization iterations') @click.option('-ds', '--dense_steps', 'opt_dense_steps', default=2000, type=int, help='Number of optimization iterations') @click.option('-v', '--video', 'opt_video', is_flag=True, help='Export a video for each dataset') @click.pass_context def cli(ctx, opt_folder_id, opt_latent_steps, opt_dense_steps, opt_video): """ The full process: - Fetch new images from the cortex - Extract labels and base latents - Extract dense embeddings - Upload extract images to the cortex """ folder = cortex_folder(opt_folder_id) files = download_cortex_files(opt_folder_id) unprocessed_files = find_unprocessed_files(files) if len(unprocessed_files) == 0: print("All files processed, nothing to do") return print("Processing folder {} ({}), {} new files".format(folder['name'], folder['id'], len(unprocessed_files))) tag = "folder_{}".format(folder['id']) paths = [file['path'] for file in unprocessed_files] find_nearest_vector_for_images( paths=paths, opt_dims=512, opt_steps=opt_dense_steps, opt_video=opt_video, opt_tag=tag, opt_limit=-1 ) params = params_dense_dict(tag) find_dense_embedding_for_images(params)