import click from app.settings import types from app.utils import click_utils from app.settings import app_cfg as cfg s3_dirs = {'media': cfg.S3_MEDIA_ROOT, 'metadata': cfg.S3_METADATA_ROOT} @click.command() @click.option('-i', '--input', 'opt_fps_in', required=True, multiple=True, help='Input directory') @click.option('--name', 'opt_dataset_name', required=True, help='Dataset key (eg "lfw"') @click.option('-a', '--action', 'opt_action', type=click.Choice(['sync', 'put']), default='sync', help='S3 action') @click.option('-t', '--type', 'opt_type', type=click.Choice(s3_dirs.keys()), required=True, help='S3 location') @click.option('--dry-run', 'opt_dryrun', is_flag=True, default=False) @click.pass_context def cli(ctx, opt_fps_in, opt_dataset_name, opt_action, opt_type, opt_dryrun): """Syncs files with S3/spaces server""" from os.path import join from pathlib import Path from tqdm import tqdm import pandas as pd import subprocess from app.utils import logger_utils, file_utils # ------------------------------------------------- # init here log = logger_utils.Logger.getLogger() for opt_fp_in in opt_fps_in: dir_dst = join(s3_dirs[opt_type], opt_dataset_name, '') if Path(opt_fp_in).is_dir(): fp_src = join(opt_fp_in, '') # add trailing slashes else: fp_src = join(opt_fp_in) cmd = ['s3cmd', opt_action, fp_src, dir_dst, '-P', '--follow-symlinks'] log.info(' '.join(cmd)) if not opt_dryrun: subprocess.call(cmd)