summaryrefslogtreecommitdiff
path: root/megapixels/commands/datasets/s3.py
diff options
context:
space:
mode:
Diffstat (limited to 'megapixels/commands/datasets/s3.py')
-rw-r--r--megapixels/commands/datasets/s3.py47
1 files changed, 0 insertions, 47 deletions
diff --git a/megapixels/commands/datasets/s3.py b/megapixels/commands/datasets/s3.py
deleted file mode 100644
index 7769896b..00000000
--- a/megapixels/commands/datasets/s3.py
+++ /dev/null
@@ -1,47 +0,0 @@
-import click
-
-from app.settings import types
-from app.utils import click_utils
-from app.settings import app_cfg as cfg
-
-s3_dirs = {'media': cfg.S3_MEDIA_ROOT, 'metadata': cfg.S3_METADATA_ROOT}
-
-@click.command()
-@click.option('-i', '--input', 'opt_fps_in', required=True, multiple=True,
- help='Input directory')
-@click.option('--name', 'opt_dataset_name', required=True,
- help='Dataset key (eg "lfw"')
-@click.option('-a', '--action', 'opt_action', type=click.Choice(['sync', 'put']), default='sync',
- help='S3 action')
-@click.option('-t', '--type', 'opt_type', type=click.Choice(s3_dirs.keys()), required=True,
- help='S3 location')
-@click.option('--dry-run', 'opt_dryrun', is_flag=True, default=False)
-@click.pass_context
-def cli(ctx, opt_fps_in, opt_dataset_name, opt_action, opt_type, opt_dryrun):
- """Syncs files with S3/spaces server"""
-
- from os.path import join
- from pathlib import Path
-
- from tqdm import tqdm
- import pandas as pd
- import subprocess
-
- from app.utils import logger_utils, file_utils
-
- # -------------------------------------------------
- # init here
-
- log = logger_utils.Logger.getLogger()
- for opt_fp_in in opt_fps_in:
- dir_dst = join(s3_dirs[opt_type], opt_dataset_name, '')
- if Path(opt_fp_in).is_dir():
- fp_src = join(opt_fp_in, '') # add trailing slashes
- else:
- fp_src = join(opt_fp_in)
- cmd = ['s3cmd', opt_action, fp_src, dir_dst, '-P', '--follow-symlinks']
- log.info(' '.join(cmd))
- if not opt_dryrun:
- subprocess.call(cmd)
-
- \ No newline at end of file