diff options
Diffstat (limited to 'megapixels/commands/datasets/download_images.py')
| -rw-r--r-- | megapixels/commands/datasets/download_images.py | 8 |
1 files changed, 5 insertions, 3 deletions
diff --git a/megapixels/commands/datasets/download_images.py b/megapixels/commands/datasets/download_images.py index 45ca8f6e..f2fdfb8f 100644 --- a/megapixels/commands/datasets/download_images.py +++ b/megapixels/commands/datasets/download_images.py @@ -10,8 +10,10 @@ import click help='Number of threads') @click.option('--wayback', 'opt_wayback', is_flag=True, default=False, help='Check Wayback archive for URL and download cached image') +@click.option('--url', 'opt_key_url', default='url', help='Field name for URL', show_default=True) +@click.option('--filepath', 'opt_key_filepath', default='filepath', help='Field name for filepath', show_default=True) @click.pass_context -def cli(ctx, opt_fp_in, opt_fp_out, opt_threads, opt_wayback): +def cli(ctx, opt_fp_in, opt_fp_out, opt_threads, opt_key_filepath, opt_key_url, opt_wayback): """Threaded image downloader""" """ @@ -69,11 +71,11 @@ def cli(ctx, opt_fp_in, opt_fp_out, opt_threads, opt_wayback): pool_items = [] log.debug(f'Initializing multithreaded pool...') for x in tqdm(records): - fp_dst = join(opt_fp_out, x['filepath']) + fp_dst = join(opt_fp_out, x[opt_key_filepath]) fp_dst_is_file = Path(fp_dst).is_file() fp_dst_is_err = Path(f'{fp_dst}_error.txt').is_file() if not fp_dst_is_file and not fp_dst_is_err: - pool_items.append({'url':x['url'], 'filepath': fp_dst, 'opt_wayback': opt_wayback}) + pool_items.append({'url':x[opt_key_url], 'filepath': fp_dst, 'opt_wayback': opt_wayback}) num_items = len(pool_items) log.info(f'Going to download {num_items:,} files') |