diff options
Diffstat (limited to 'megapixels/commands/cv')
| -rw-r--r-- | megapixels/commands/cv/gen_face_vec.py (renamed from megapixels/commands/cv/rois_to_vecs.py) | 56 | ||||
| -rw-r--r-- | megapixels/commands/cv/gen_pose.py (renamed from megapixels/commands/cv/rois_to_pose.py) | 72 | ||||
| -rw-r--r-- | megapixels/commands/cv/gen_rois.py (renamed from megapixels/commands/cv/files_to_rois.py) | 52 |
3 files changed, 112 insertions, 68 deletions
diff --git a/megapixels/commands/cv/rois_to_vecs.py b/megapixels/commands/cv/gen_face_vec.py index 525f4404..83e1460d 100644 --- a/megapixels/commands/cv/rois_to_vecs.py +++ b/megapixels/commands/cv/gen_face_vec.py @@ -9,14 +9,20 @@ from app.utils import click_utils from app.settings import app_cfg as cfg @click.command() -@click.option('-i', '--input', 'opt_fp_files', required=True, - help='Input file meta CSV') -@click.option('-r', '--rois', 'opt_fp_rois', required=True, - help='Input ROI CSV') -@click.option('-m', '--media', 'opt_dir_media', required=True, - help='Input media directory') -@click.option('-o', '--output', 'opt_fp_out', required=True, - help='Output CSV') +@click.option('-o', '--output', 'opt_fp_out', default=None, + help='Override enum output filename CSV') +@click.option('-m', '--media', 'opt_dir_media', default=None, + help='Override enum media directory') +@click.option('--data_store', 'opt_data_store', + type=cfg.DataStoreVar, + default=click_utils.get_default(types.DataStore.SSD), + show_default=True, + help=click_utils.show_help(types.Dataset)) +@click.option('--dataset', 'opt_dataset', + type=cfg.DatasetVar, + required=True, + show_default=True, + help=click_utils.show_help(types.Dataset)) @click.option('--size', 'opt_size', type=(int, int), default=(300, 300), help='Output image size') @@ -31,7 +37,7 @@ from app.settings import app_cfg as cfg @click.option('-g', '--gpu', 'opt_gpu', default=0, help='GPU index') @click.pass_context -def cli(ctx, opt_fp_files, opt_fp_rois, opt_dir_media, opt_fp_out, opt_size, +def cli(ctx, opt_fp_out, opt_dir_media, opt_data_store, opt_dataset, opt_size, opt_slice, opt_force, opt_gpu, opt_jitters, opt_padding): """Converts face ROIs to vectors""" @@ -48,6 +54,7 @@ def cli(ctx, opt_fp_files, opt_fp_rois, opt_dir_media, opt_fp_out, opt_size, import pandas as pd from app.models.bbox import BBox + from app.models.data_store import DataStore from app.utils import logger_utils, file_utils, im_utils from app.processors import face_recognition @@ -56,24 +63,28 @@ def cli(ctx, opt_fp_files, opt_fp_rois, opt_dir_media, opt_fp_out, opt_size, # init here log = logger_utils.Logger.getLogger() + # set data_store + data_store = DataStore(opt_data_store, opt_dataset) + + # get filepath out + fp_out = data_store.metadata(types.Metadata.FACE_VECTOR) if opt_fp_out is None else opt_fp_out + if not opt_force and Path(fp_out).exists(): + log.error('File exists. Use "-f / --force" to overwite') + return # init face processors facerec = face_recognition.RecognitionDLIB() # load data - df_file_meta = pd.read_csv(opt_fp_files) - df_rois = pd.read_csv(opt_fp_rois) + df_file = pd.read_csv(data_store.metadata(types.Metadata.FILEPATH)).set_index('index') + df_roi = pd.read_csv(data_store.metadata(types.Metadata.FACE_ROI)).set_index('index') - if not opt_force and Path(opt_fp_out).exists(): - log.error('File exists. Use "-f / --force" to overwite') - return - if opt_slice: - df_rois = df_rois[opt_slice[0]:opt_slice[1]] + df_roi = df_roi[opt_slice[0]:opt_slice[1]] # ------------------------------------------------- # process here - df_img_groups = df_rois.groupby('image_index') + df_img_groups = df_roi.groupby('image_index') log.debug('processing {:,} groups'.format(len(df_img_groups))) vecs = [] @@ -81,8 +92,11 @@ def cli(ctx, opt_fp_files, opt_fp_rois, opt_dir_media, opt_fp_out, opt_size, for image_index, df_img_group in tqdm(df_img_groups): # make fp roi_index = df_img_group.index.values[0] - file_meta = df_file_meta.iloc[image_index] # locate image meta - fp_im = join(opt_dir_media, file_meta.subdir, '{}.{}'.format(file_meta.fn, file_meta.ext)) + log.debug(f'roi_index: {roi_index}, image_index: {image_index}') + ds_file = df_file.loc[roi_index] # locate image meta + #ds_file = df_file.loc['index', image_index] # locate image meta + + fp_im = data_store.face_image(str(ds_file.subdir), str(ds_file.fn), str(ds_file.ext)) im = cv.imread(fp_im) # get bbox x = df_img_group.x.values[0] @@ -103,7 +117,7 @@ def cli(ctx, opt_fp_files, opt_fp_rois, opt_dir_media, opt_fp_out, opt_size, # save date - file_utils.mkdirs(opt_fp_out) df = pd.DataFrame.from_dict(vecs) df.index.name = 'index' - df.to_csv(opt_fp_out)
\ No newline at end of file + #file_utils.mkdirs(fp_out) + #df.to_csv(fp_out)
\ No newline at end of file diff --git a/megapixels/commands/cv/rois_to_pose.py b/megapixels/commands/cv/gen_pose.py index 3877cecf..aefadb00 100644 --- a/megapixels/commands/cv/rois_to_pose.py +++ b/megapixels/commands/cv/gen_pose.py @@ -9,14 +9,22 @@ from app.utils import click_utils from app.settings import app_cfg as cfg @click.command() -@click.option('-i', '--input', 'opt_fp_files', required=True, - help='Input ROI CSV') -@click.option('-r', '--rois', 'opt_fp_rois', required=True, - help='Input ROI CSV') -@click.option('-m', '--media', 'opt_dir_media', required=True, - help='Input media directory') -@click.option('-o', '--output', 'opt_fp_out', required=True, - help='Output CSV') +@click.option('-i', '--input', 'opt_fp_in', default=None, + help='Override enum input filename CSV') +@click.option('-o', '--output', 'opt_fp_out', default=None, + help='Override enum output filename CSV') +@click.option('-m', '--media', 'opt_dir_media', default=None, + help='Override enum media directory') +@click.option('--data_store', 'opt_data_store', + type=cfg.DataStoreVar, + default=click_utils.get_default(types.DataStore.SSD), + show_default=True, + help=click_utils.show_help(types.Dataset)) +@click.option('--dataset', 'opt_dataset', + type=cfg.DatasetVar, + required=True, + show_default=True, + help=click_utils.show_help(types.Dataset)) @click.option('--size', 'opt_size', type=(int, int), default=(300, 300), help='Output image size') @@ -27,7 +35,7 @@ from app.settings import app_cfg as cfg @click.option('-d', '--display', 'opt_display', is_flag=True, help='Display image for debugging') @click.pass_context -def cli(ctx, opt_fp_files, opt_fp_rois, opt_dir_media, opt_fp_out, opt_size, +def cli(ctx, opt_fp_in, opt_fp_out, opt_dir_media, opt_data_store, opt_dataset, opt_size, opt_slice, opt_force, opt_display): """Converts ROIs to pose: roll, yaw, pitch""" @@ -47,42 +55,48 @@ def cli(ctx, opt_fp_files, opt_fp_rois, opt_dir_media, opt_fp_out, opt_size, from app.utils import logger_utils, file_utils, im_utils from app.processors.face_landmarks import LandmarksDLIB from app.processors.face_pose import FacePoseDLIB + from app.models.data_store import DataStore # ------------------------------------------------- # init here log = logger_utils.Logger.getLogger() + + # set data_store + data_store = DataStore(opt_data_store, opt_dataset) + + # get filepath out + fp_out = data_store.metadata(types.Metadata.FACE_POSE) if opt_fp_out is None else opt_fp_out + if not opt_force and Path(fp_out).exists(): + log.error('File exists. Use "-f / --force" to overwite') + return # init face processors face_pose = FacePoseDLIB() face_landmarks = LandmarksDLIB() - # load datra - df_files = pd.read_csv(opt_fp_files) - df_rois = pd.read_csv(opt_fp_rois) - - if not opt_force and Path(opt_fp_out).exists(): - log.error('File exists. Use "-f / --force" to overwite') - return - + # load filepath data + fp_filepath = data_store.metadata(types.Metadata.FILEPATH) + df_filepath = pd.read_csv(fp_filepath) + # load ROI data + fp_roi = data_store.metadata(types.Metadata.FACE_ROI) + df_roi = pd.read_csv(fp_roi) + # slice if you want if opt_slice: - df_rois = df_rois[opt_slice[0]:opt_slice[1]] - - # ------------------------------------------------- - # process here - df_img_groups = df_rois.groupby('image_index') + df_roi = df_roi[opt_slice[0]:opt_slice[1]] + # group by image index (speedup if multiple faces per image) + df_img_groups = df_roi.groupby('image_index') log.debug('processing {:,} groups'.format(len(df_img_groups))) - + # store poses and convert to DataFrame poses = [] # iterate - #for df_roi_group_idx, df_roi_group in tqdm(df_roi_groups): for image_index, df_img_group in tqdm(df_img_groups): # make fp - #image_index = df_roi_group.image_index.values[0] - pds_file = df_files.iloc[image_index] - fp_im = join(opt_dir_media, pds_file.subdir, '{}.{}'.format(pds_file.fn, pds_file.ext)) + ds_file = df_filepath.iloc[image_index] + fp_im = data_store.face_image(ds_file.subdir, ds_file.fn, ds_file.ext) + #fp_im = join(opt_dir_media, ds_file.subdir, '{}.{}'.format(ds_file.fn, ds_file.ext)) im = cv.imread(fp_im) # get bbox x = df_img_group.x.values[0] @@ -121,7 +135,7 @@ def cli(ctx, opt_fp_files, opt_fp_rois, opt_dir_media, opt_fp_out, opt_size, # save date - file_utils.mkdirs(opt_fp_out) + file_utils.mkdirs(fp_out) df = pd.DataFrame.from_dict(poses) df.index.name = 'index' - df.to_csv(opt_fp_out)
\ No newline at end of file + df.to_csv(fp_out)
\ No newline at end of file diff --git a/megapixels/commands/cv/files_to_rois.py b/megapixels/commands/cv/gen_rois.py index 1aaf991c..20dd598a 100644 --- a/megapixels/commands/cv/files_to_rois.py +++ b/megapixels/commands/cv/gen_rois.py @@ -12,12 +12,22 @@ from app.settings import app_cfg as cfg color_filters = {'color': 1, 'gray': 2, 'all': 3} @click.command() -@click.option('-i', '--input', 'opt_fp_in', required=True, - help='Input CSV (eg image_files.csv)') -@click.option('-m', '--media', 'opt_dir_media', required=True, - help='Input media directory') -@click.option('-o', '--output', 'opt_fp_out', required=True, - help='Output CSV') +@click.option('-i', '--input', 'opt_fp_in', default=None, + help='Override enum input filename CSV') +@click.option('-o', '--output', 'opt_fp_out', default=None, + help='Override enum output filename CSV') +@click.option('-m', '--media', 'opt_dir_media', default=None, + help='Override enum media directory') +@click.option('--data_store', 'opt_data_store', + type=cfg.DataStoreVar, + default=click_utils.get_default(types.DataStore.SSD), + show_default=True, + help=click_utils.show_help(types.Dataset)) +@click.option('--dataset', 'opt_dataset', + type=cfg.DatasetVar, + required=True, + show_default=True, + help=click_utils.show_help(types.Dataset)) @click.option('--size', 'opt_size', type=(int, int), default=(300, 300), help='Output image size') @@ -40,10 +50,10 @@ color_filters = {'color': 1, 'gray': 2, 'all': 3} @click.option('--color', 'opt_color_filter', type=click.Choice(color_filters.keys()), default='all', help='Filter to keep color or grayscale images (color = keep color') -@click.option('--largest', 'opt_largest', is_flag=True, +@click.option('--largest/--all-faces', 'opt_largest', is_flag=True, default=True, help='Only keep largest face') @click.pass_context -def cli(ctx, opt_fp_in, opt_dir_media, opt_fp_out, opt_size, opt_detector_type, +def cli(ctx, opt_fp_in, opt_dir_media, opt_fp_out, opt_data_store, opt_dataset, opt_size, opt_detector_type, opt_gpu, opt_conf_thresh, opt_pyramids, opt_slice, opt_display, opt_force, opt_color_filter, opt_largest): """Converts frames with faces to CSV of ROIs""" @@ -61,17 +71,24 @@ def cli(ctx, opt_fp_in, opt_dir_media, opt_fp_out, opt_size, opt_detector_type, import pandas as pd from app.utils import logger_utils, file_utils, im_utils - from app.processors import face_detector + from app.processors import face_detector + from app.models.data_store import DataStore # ------------------------------------------------- # init here log = logger_utils.Logger.getLogger() - if not opt_force and Path(opt_fp_out).exists(): + # set data_store + data_store = DataStore(opt_data_store, opt_dataset) + + # get filepath out + fp_out = data_store.metadata(types.Metadata.FACE_ROI) if opt_fp_out is None else opt_fp_out + if not opt_force and Path(fp_out).exists(): log.error('File exists. Use "-f / --force" to overwite') return + # set detector if opt_detector_type == types.FaceDetectNet.CVDNN: detector = face_detector.DetectorCVDNN() elif opt_detector_type == types.FaceDetectNet.DLIB_CNN: @@ -85,22 +102,21 @@ def cli(ctx, opt_fp_in, opt_dir_media, opt_fp_out, opt_size, opt_detector_type, return - # ------------------------------------------------- - # process here - color_filter = color_filters[opt_color_filter] - # get list of files to process - df_files = pd.read_csv(opt_fp_in).set_index('index') - + fp_in = data_store.metadata(types.Metadata.FILEPATH) if opt_fp_in is None else opt_fp_in + df_files = pd.read_csv(fp_in).set_index('index') if opt_slice: df_files = df_files[opt_slice[0]:opt_slice[1]] log.debug('processing {:,} files'.format(len(df_files))) + # filter out grayscale + color_filter = color_filters[opt_color_filter] data = [] for df_file in tqdm(df_files.itertuples(), total=len(df_files)): - fp_im = join(opt_dir_media, str(df_file.subdir), f'{df_file.fn}.{df_file.ext}') + fp_im = data_store.face_image(str(df_file.subdir), str(df_file.fn), str(df_file.ext)) + #fp_im = join(opt_dir_media, str(df_file.subdir), f'{df_file.fn}.{df_file.ext}') im = cv.imread(fp_im) # filter out color or grayscale iamges @@ -150,7 +166,7 @@ def cli(ctx, opt_fp_in, opt_dir_media, opt_fp_out, opt_size, opt_detector_type, break # save date - file_utils.mkdirs(opt_fp_out) + file_utils.mkdirs(fp_out) df = pd.DataFrame.from_dict(data) df.index.name = 'index' df.to_csv(opt_fp_out)
\ No newline at end of file |