still sorting CSV vectors indexes

3 files changed, 112 insertions, 68 deletions

diff --git a/megapixels/commands/cv/rois_to_vecs.py b/megapixels/commands/cv/gen_face_vec.py
index 525f4404..83e1460d 100644
--- a/megapixels/commands/cv/rois_to_vecs.py
+++ b/megapixels/commands/cv/gen_face_vec.py
@@ -9,14 +9,20 @@ from app.utils import click_utils
 from app.settings import app_cfg as cfg
 
 @click.command()
-@click.option('-i', '--input', 'opt_fp_files', required=True,
-  help='Input file meta CSV')
-@click.option('-r', '--rois', 'opt_fp_rois', required=True,
-  help='Input ROI CSV')
-@click.option('-m', '--media', 'opt_dir_media', required=True,
-  help='Input media directory')
-@click.option('-o', '--output', 'opt_fp_out', required=True,
-  help='Output CSV')
+@click.option('-o', '--output', 'opt_fp_out', default=None,
+  help='Override enum output filename CSV')
+@click.option('-m', '--media', 'opt_dir_media', default=None,
+  help='Override enum media directory')
+@click.option('--data_store', 'opt_data_store',
+  type=cfg.DataStoreVar,
+  default=click_utils.get_default(types.DataStore.SSD),
+  show_default=True,
+  help=click_utils.show_help(types.Dataset))
+@click.option('--dataset', 'opt_dataset',
+  type=cfg.DatasetVar,
+  required=True,
+  show_default=True,
+  help=click_utils.show_help(types.Dataset))
 @click.option('--size', 'opt_size', 
   type=(int, int), default=(300, 300),
   help='Output image size')
@@ -31,7 +37,7 @@ from app.settings import app_cfg as cfg
 @click.option('-g', '--gpu', 'opt_gpu', default=0,
   help='GPU index')
 @click.pass_context
-def cli(ctx, opt_fp_files, opt_fp_rois, opt_dir_media, opt_fp_out, opt_size, 
+def cli(ctx, opt_fp_out, opt_dir_media, opt_data_store, opt_dataset, opt_size, 
   opt_slice, opt_force, opt_gpu, opt_jitters, opt_padding):
   """Converts face ROIs to vectors"""
   
@@ -48,6 +54,7 @@ def cli(ctx, opt_fp_files, opt_fp_rois, opt_dir_media, opt_fp_out, opt_size,
   import pandas as pd
 
   from app.models.bbox import BBox
+  from app.models.data_store import DataStore
   from app.utils import logger_utils, file_utils, im_utils
   from app.processors import face_recognition
 
@@ -56,24 +63,28 @@ def cli(ctx, opt_fp_files, opt_fp_rois, opt_dir_media, opt_fp_out, opt_size,
   # init here
 
   log = logger_utils.Logger.getLogger()
+  # set data_store
+  data_store = DataStore(opt_data_store, opt_dataset)
+
+  # get filepath out
+  fp_out = data_store.metadata(types.Metadata.FACE_VECTOR) if opt_fp_out is None else opt_fp_out
+  if not opt_force and Path(fp_out).exists():
+    log.error('File exists. Use "-f / --force" to overwite')
+    return
 
   # init face processors
   facerec = face_recognition.RecognitionDLIB()
 
   # load data
-  df_file_meta = pd.read_csv(opt_fp_files)
-  df_rois = pd.read_csv(opt_fp_rois)
+  df_file = pd.read_csv(data_store.metadata(types.Metadata.FILEPATH)).set_index('index')
+  df_roi = pd.read_csv(data_store.metadata(types.Metadata.FACE_ROI)).set_index('index')
 
-  if not opt_force and Path(opt_fp_out).exists():
-    log.error('File exists. Use "-f / --force" to overwite')
-    return
-  
   if opt_slice:
-    df_rois = df_rois[opt_slice[0]:opt_slice[1]]
+    df_roi = df_roi[opt_slice[0]:opt_slice[1]]
   
   # -------------------------------------------------
   # process here
-  df_img_groups = df_rois.groupby('image_index')
+  df_img_groups = df_roi.groupby('image_index')
   log.debug('processing {:,} groups'.format(len(df_img_groups)))
 
   vecs = []
@@ -81,8 +92,11 @@ def cli(ctx, opt_fp_files, opt_fp_rois, opt_dir_media, opt_fp_out, opt_size,
   for image_index, df_img_group in tqdm(df_img_groups):
     # make fp
     roi_index = df_img_group.index.values[0]
-    file_meta = df_file_meta.iloc[image_index]  # locate image meta
-    fp_im = join(opt_dir_media, file_meta.subdir, '{}.{}'.format(file_meta.fn, file_meta.ext))
+    log.debug(f'roi_index: {roi_index}, image_index: {image_index}')
+    ds_file = df_file.loc[roi_index]  # locate image meta
+    #ds_file = df_file.loc['index', image_index]  # locate image meta
+
+    fp_im = data_store.face_image(str(ds_file.subdir), str(ds_file.fn), str(ds_file.ext))
     im = cv.imread(fp_im)
     # get bbox
     x = df_img_group.x.values[0]
@@ -103,7 +117,7 @@ def cli(ctx, opt_fp_files, opt_fp_rois, opt_dir_media, opt_fp_out, opt_size,
 
 
   # save date
-  file_utils.mkdirs(opt_fp_out)
   df = pd.DataFrame.from_dict(vecs)
   df.index.name = 'index'
-  df.to_csv(opt_fp_out)
\ No newline at end of file
+  #file_utils.mkdirs(fp_out)
+  #df.to_csv(fp_out)
\ No newline at end of file
diff --git a/megapixels/commands/cv/rois_to_pose.py b/megapixels/commands/cv/gen_pose.py
index 3877cecf..aefadb00 100644
--- a/megapixels/commands/cv/rois_to_pose.py
+++ b/megapixels/commands/cv/gen_pose.py
@@ -9,14 +9,22 @@ from app.utils import click_utils
 from app.settings import app_cfg as cfg
 
 @click.command()
-@click.option('-i', '--input', 'opt_fp_files', required=True,
-  help='Input ROI CSV')
-@click.option('-r', '--rois', 'opt_fp_rois', required=True,
-  help='Input ROI CSV')
-@click.option('-m', '--media', 'opt_dir_media', required=True,
-  help='Input media directory')
-@click.option('-o', '--output', 'opt_fp_out', required=True,
-  help='Output CSV')
+@click.option('-i', '--input', 'opt_fp_in', default=None,
+  help='Override enum input filename CSV')
+@click.option('-o', '--output', 'opt_fp_out', default=None,
+  help='Override enum output filename CSV')
+@click.option('-m', '--media', 'opt_dir_media', default=None,
+  help='Override enum media directory')
+@click.option('--data_store', 'opt_data_store',
+  type=cfg.DataStoreVar,
+  default=click_utils.get_default(types.DataStore.SSD),
+  show_default=True,
+  help=click_utils.show_help(types.Dataset))
+@click.option('--dataset', 'opt_dataset',
+  type=cfg.DatasetVar,
+  required=True,
+  show_default=True,
+  help=click_utils.show_help(types.Dataset))
 @click.option('--size', 'opt_size', 
   type=(int, int), default=(300, 300),
   help='Output image size')
@@ -27,7 +35,7 @@ from app.settings import app_cfg as cfg
 @click.option('-d', '--display', 'opt_display', is_flag=True,
   help='Display image for debugging')
 @click.pass_context
-def cli(ctx, opt_fp_files, opt_fp_rois, opt_dir_media, opt_fp_out, opt_size, 
+def cli(ctx, opt_fp_in, opt_fp_out, opt_dir_media, opt_data_store, opt_dataset, opt_size, 
   opt_slice, opt_force, opt_display):
   """Converts ROIs to pose: roll, yaw, pitch"""
   
@@ -47,42 +55,48 @@ def cli(ctx, opt_fp_files, opt_fp_rois, opt_dir_media, opt_fp_out, opt_size,
   from app.utils import logger_utils, file_utils, im_utils
   from app.processors.face_landmarks import LandmarksDLIB
   from app.processors.face_pose import FacePoseDLIB
+  from app.models.data_store import DataStore
 
   # -------------------------------------------------
   # init here
 
   log = logger_utils.Logger.getLogger()
+  
+  # set data_store
+  data_store = DataStore(opt_data_store, opt_dataset)
+
+  # get filepath out
+  fp_out = data_store.metadata(types.Metadata.FACE_POSE) if opt_fp_out is None else opt_fp_out
+  if not opt_force and Path(fp_out).exists():
+    log.error('File exists. Use "-f / --force" to overwite')
+    return
 
   # init face processors
   face_pose = FacePoseDLIB()
   face_landmarks = LandmarksDLIB()
 
-  # load datra
-  df_files = pd.read_csv(opt_fp_files)
-  df_rois = pd.read_csv(opt_fp_rois)
-
-  if not opt_force and Path(opt_fp_out).exists():
-    log.error('File exists. Use "-f / --force" to overwite')
-    return
-  
+  # load filepath data
+  fp_filepath = data_store.metadata(types.Metadata.FILEPATH)
+  df_filepath = pd.read_csv(fp_filepath)
+  # load ROI data
+  fp_roi = data_store.metadata(types.Metadata.FACE_ROI)
+  df_roi = pd.read_csv(fp_roi)
+  # slice if you want
   if opt_slice:
-    df_rois = df_rois[opt_slice[0]:opt_slice[1]]
-  
-  # -------------------------------------------------
-  # process here
-  df_img_groups = df_rois.groupby('image_index')
+    df_roi = df_roi[opt_slice[0]:opt_slice[1]]
+  # group by image index (speedup if multiple faces per image)
+  df_img_groups = df_roi.groupby('image_index')
   log.debug('processing {:,} groups'.format(len(df_img_groups)))
 
-
+  # store poses and convert to DataFrame
   poses = []
 
   # iterate 
-  #for df_roi_group_idx, df_roi_group in tqdm(df_roi_groups):
   for image_index, df_img_group in tqdm(df_img_groups):
     # make fp
-    #image_index = df_roi_group.image_index.values[0]
-    pds_file = df_files.iloc[image_index]
-    fp_im = join(opt_dir_media, pds_file.subdir, '{}.{}'.format(pds_file.fn, pds_file.ext))
+    ds_file = df_filepath.iloc[image_index]
+    fp_im = data_store.face_image(ds_file.subdir, ds_file.fn, ds_file.ext)
+    #fp_im = join(opt_dir_media, ds_file.subdir, '{}.{}'.format(ds_file.fn, ds_file.ext))
     im = cv.imread(fp_im)
     # get bbox
     x = df_img_group.x.values[0]
@@ -121,7 +135,7 @@ def cli(ctx, opt_fp_files, opt_fp_rois, opt_dir_media, opt_fp_out, opt_size,
 
 
   # save date
-  file_utils.mkdirs(opt_fp_out)
+  file_utils.mkdirs(fp_out)
   df = pd.DataFrame.from_dict(poses)
   df.index.name = 'index'
-  df.to_csv(opt_fp_out)
\ No newline at end of file
+  df.to_csv(fp_out)
\ No newline at end of file
diff --git a/megapixels/commands/cv/files_to_rois.py b/megapixels/commands/cv/gen_rois.py
index 1aaf991c..20dd598a 100644
--- a/megapixels/commands/cv/files_to_rois.py
+++ b/megapixels/commands/cv/gen_rois.py
@@ -12,12 +12,22 @@ from app.settings import app_cfg as cfg
 color_filters = {'color': 1, 'gray': 2, 'all': 3}
 
 @click.command()
-@click.option('-i', '--input', 'opt_fp_in', required=True,
-  help='Input CSV (eg image_files.csv)')
-@click.option('-m', '--media', 'opt_dir_media', required=True,
-  help='Input media directory')
-@click.option('-o', '--output', 'opt_fp_out', required=True,
-  help='Output CSV')
+@click.option('-i', '--input', 'opt_fp_in', default=None,
+  help='Override enum input filename CSV')
+@click.option('-o', '--output', 'opt_fp_out', default=None,
+  help='Override enum output filename CSV')
+@click.option('-m', '--media', 'opt_dir_media', default=None,
+  help='Override enum media directory')
+@click.option('--data_store', 'opt_data_store',
+  type=cfg.DataStoreVar,
+  default=click_utils.get_default(types.DataStore.SSD),
+  show_default=True,
+  help=click_utils.show_help(types.Dataset))
+@click.option('--dataset', 'opt_dataset',
+  type=cfg.DatasetVar,
+  required=True,
+  show_default=True,
+  help=click_utils.show_help(types.Dataset))
 @click.option('--size', 'opt_size', 
   type=(int, int), default=(300, 300),
   help='Output image size')
@@ -40,10 +50,10 @@ color_filters = {'color': 1, 'gray': 2, 'all': 3}
 @click.option('--color', 'opt_color_filter', 
   type=click.Choice(color_filters.keys()), default='all',
   help='Filter to keep color or grayscale images (color = keep color')
-@click.option('--largest', 'opt_largest', is_flag=True, 
+@click.option('--largest/--all-faces', 'opt_largest', is_flag=True, default=True,
   help='Only keep largest face')
 @click.pass_context
-def cli(ctx, opt_fp_in, opt_dir_media, opt_fp_out, opt_size, opt_detector_type, 
+def cli(ctx, opt_fp_in, opt_dir_media, opt_fp_out, opt_data_store, opt_dataset, opt_size, opt_detector_type, 
   opt_gpu, opt_conf_thresh, opt_pyramids, opt_slice, opt_display, opt_force, opt_color_filter,
   opt_largest):
   """Converts frames with faces to CSV of ROIs"""
@@ -61,17 +71,24 @@ def cli(ctx, opt_fp_in, opt_dir_media, opt_fp_out, opt_size, opt_detector_type,
   import pandas as pd
 
   from app.utils import logger_utils, file_utils, im_utils
-  from app.processors import face_detector 
+  from app.processors import face_detector
+  from app.models.data_store import DataStore
 
   # -------------------------------------------------
   # init here
 
   log = logger_utils.Logger.getLogger()
 
-  if not opt_force and Path(opt_fp_out).exists():
+  # set data_store
+  data_store = DataStore(opt_data_store, opt_dataset)
+
+  # get filepath out
+  fp_out = data_store.metadata(types.Metadata.FACE_ROI) if opt_fp_out is None else opt_fp_out
+  if not opt_force and Path(fp_out).exists():
     log.error('File exists. Use "-f / --force" to overwite')
     return
   
+  # set detector
   if opt_detector_type == types.FaceDetectNet.CVDNN:
     detector = face_detector.DetectorCVDNN()
   elif opt_detector_type == types.FaceDetectNet.DLIB_CNN:
@@ -85,22 +102,21 @@ def cli(ctx, opt_fp_in, opt_dir_media, opt_fp_out, opt_size, opt_detector_type,
     return
 
   
-  # -------------------------------------------------
-  # process here
-  color_filter = color_filters[opt_color_filter]
-  
   # get list of files to process
-  df_files = pd.read_csv(opt_fp_in).set_index('index')
-
+  fp_in = data_store.metadata(types.Metadata.FILEPATH) if opt_fp_in is None else opt_fp_in
+  df_files = pd.read_csv(fp_in).set_index('index')
   if opt_slice:
     df_files = df_files[opt_slice[0]:opt_slice[1]]
   log.debug('processing {:,} files'.format(len(df_files)))
 
+  # filter out grayscale
+  color_filter = color_filters[opt_color_filter]
 
   data = []
 
   for df_file in tqdm(df_files.itertuples(), total=len(df_files)):
-    fp_im = join(opt_dir_media, str(df_file.subdir), f'{df_file.fn}.{df_file.ext}')
+    fp_im = data_store.face_image(str(df_file.subdir), str(df_file.fn), str(df_file.ext))
+    #fp_im = join(opt_dir_media, str(df_file.subdir), f'{df_file.fn}.{df_file.ext}')
     im = cv.imread(fp_im)
 
     # filter out color or grayscale iamges
@@ -150,7 +166,7 @@ def cli(ctx, opt_fp_in, opt_dir_media, opt_fp_out, opt_size, opt_detector_type,
           break
 
   # save date
-  file_utils.mkdirs(opt_fp_out)
+  file_utils.mkdirs(fp_out)
   df = pd.DataFrame.from_dict(data)
   df.index.name = 'index'
   df.to_csv(opt_fp_out)
\ No newline at end of file