From 88ec48e1c4d93ba9cd3aa186c068ef2aa4c27c56 Mon Sep 17 00:00:00 2001 From: adamhrv Date: Mon, 17 Dec 2018 01:37:31 +0100 Subject: fixing dataset procesosrs --- megapixels/app/models/data_store.py | 20 +- megapixels/app/models/dataset.py | 127 +++-- megapixels/app/settings/app_cfg.py | 2 +- megapixels/app/settings/types.py | 2 +- megapixels/app/utils/file_utils.py | 12 +- megapixels/commands/cv/cluster.py | 22 +- megapixels/commands/cv/face_pose.py | 140 ++++++ megapixels/commands/cv/face_roi.py | 171 +++++++ megapixels/commands/cv/face_vector.py | 125 +++++ megapixels/commands/cv/gen_face_vec.py | 123 ----- megapixels/commands/cv/gen_pose.py | 141 ------ megapixels/commands/cv/gen_rois.py | 172 ------- megapixels/commands/datasets/filter_by_pose.py | 41 +- megapixels/commands/datasets/gen_filepath.py | 4 +- megapixels/commands/datasets/gen_sha256.py | 152 ------ megapixels/commands/datasets/gen_uuid.py | 2 +- megapixels/commands/datasets/identity_meta_lfw.py | 93 ++++ .../commands/datasets/identity_meta_vgg_face2.py | 88 ++++ megapixels/commands/datasets/lookup.py | 9 +- megapixels/commands/datasets/records.py | 159 +++++++ megapixels/commands/datasets/s3.py | 47 -- megapixels/commands/datasets/s3_sync.py | 57 +++ megapixels/commands/datasets/symlink.py | 45 -- megapixels/commands/datasets/symlink_uuid.py | 57 +++ megapixels/commands/demo/face_search.py | 3 +- megapixels/notebooks/_local_scratch.ipynb | 196 ++++---- .../datasets/lfw/lfw_make_identity_csv.ipynb | 510 +++++++++++++++++++++ .../notebooks/examples/face_recognition_demo.ipynb | 4 +- 28 files changed, 1653 insertions(+), 871 deletions(-) create mode 100644 megapixels/commands/cv/face_pose.py create mode 100644 megapixels/commands/cv/face_roi.py create mode 100644 megapixels/commands/cv/face_vector.py delete mode 100644 megapixels/commands/cv/gen_face_vec.py delete mode 100644 megapixels/commands/cv/gen_pose.py delete mode 100644 megapixels/commands/cv/gen_rois.py delete mode 100644 megapixels/commands/datasets/gen_sha256.py create mode 100644 megapixels/commands/datasets/identity_meta_lfw.py create mode 100644 megapixels/commands/datasets/identity_meta_vgg_face2.py create mode 100644 megapixels/commands/datasets/records.py delete mode 100644 megapixels/commands/datasets/s3.py create mode 100644 megapixels/commands/datasets/s3_sync.py delete mode 100644 megapixels/commands/datasets/symlink.py create mode 100644 megapixels/commands/datasets/symlink_uuid.py create mode 100644 megapixels/notebooks/datasets/lfw/lfw_make_identity_csv.ipynb diff --git a/megapixels/app/models/data_store.py b/megapixels/app/models/data_store.py index 8ec1f8ba..244aba60 100644 --- a/megapixels/app/models/data_store.py +++ b/megapixels/app/models/data_store.py @@ -21,15 +21,27 @@ class DataStore: def metadata(self, enum_type): return join(self.dir_metadata, f'{enum_type.name.lower()}.csv') + def metadata(self, enum_type): + return join(self.dir_metadata) + def media_images_original(self): return join(self.dir_media, 'original') - def face_image(self, subdir, fn, ext): + def face(self, subdir, fn, ext): return join(self.dir_media, 'original', subdir, f'{fn}.{ext}') - def face_image_crop(self, subdir, fn, ext): + def face_crop(self, subdir, fn, ext): return join(self.dir_media, 'cropped', subdir, f'{fn}.{ext}') + def face_uuid(self, uuid, ext): + return join(self.dir_media, 'uuid',f'{uuid}.{ext}') + + def face_crop_uuid(self, uuid, ext): + return join(self.dir_media, 'uuid', f'{uuid}.{ext}') + + def uuid_dir(self): + return join(self.dir_media, 'uuid') + class DataStoreS3: # S3 server @@ -40,11 +52,11 @@ class DataStoreS3: def metadata(self, opt_metadata_type, ext='csv'): return join(self._dir_metadata, f'{opt_metadata_type.name.lower()}.{ext}') - def face_image(self, opt_uuid, ext='jpg'): + def face(self, opt_uuid, ext='jpg'): #return join(self._dir_media, 'original', f'{opt_uuid}.{ext}') return join(self._dir_media, f'{opt_uuid}.{ext}') - def face_image_crop(self, opt_uuid, ext='jpg'): + def face_crop(self, opt_uuid, ext='jpg'): # not currently using? return join(self._dir_media, 'cropped', f'{opt_uuid}.{ext}') diff --git a/megapixels/app/models/dataset.py b/megapixels/app/models/dataset.py index 8fef8a7e..35e10465 100644 --- a/megapixels/app/models/dataset.py +++ b/megapixels/app/models/dataset.py @@ -23,7 +23,7 @@ from app.utils.logger_utils import Logger class Dataset: - def __init__(self, opt_data_store, opt_dataset_type, load_files=True): + def __init__(self, opt_data_store, opt_dataset_type): self._dataset_type = opt_dataset_type # enum type self.log = Logger.getLogger() self._metadata = {} @@ -31,31 +31,62 @@ class Dataset: self._nullframe = pd.DataFrame() # empty placeholder self.data_store = DataStore(opt_data_store, self._dataset_type) self.data_store_s3 = DataStoreS3(self._dataset_type) - self.load_metadata() - - def load_metadata(self): - '''Loads all CSV files into (dict) of DataFrames''' - self.log.info(f'creating dataset: {self._dataset_type}...') - for metadata_type in types.Metadata: - self.log.info(f'load metadata: {metadata_type}') - fp_csv = self.data_store.metadata(metadata_type) - self.log.info(f'loading: {fp_csv}') - if Path(fp_csv).is_file(): - self._metadata[metadata_type] = pd.read_csv(fp_csv).set_index('index') - if metadata_type == types.Metadata.FACE_VECTOR: - # convert DataFrame to list of floats - self._face_vectors = self.df_to_vec_list(self._metadata[metadata_type]) - self.log.info(f'build face vector dict: {len(self._face_vectors)}') - self._metadata[metadata_type].drop('vec', axis=1, inplace=True) - else: - self.log.error(f'File not found: {fp_csv}. Exiting.') - sys.exit() - self.log.info('finished loading') + + def load_face_vectors(self): + metadata_type = types.Metadata.FACE_VECTOR + fp_csv = self.data_store.metadata(metadata_type) + self.log.info(f'loading: {fp_csv}') + if Path(fp_csv).is_file(): + self._metadata[metadata_type] = pd.read_csv(fp_csv).set_index('index') + # convert DataFrame to list of floats + self._face_vectors = self.df_vecs_to_dict(self._metadata[metadata_type]) + self._face_vector_idxs = self.df_vec_idxs_to_dict(self._metadata[metadata_type]) + self.log.info(f'build face vector dict: {len(self._face_vectors)}') + # remove the face vector column, it can be several GB of memory + self._metadata[metadata_type].drop('vec', axis=1, inplace=True) + else: + self.log.error(f'File not found: {fp_csv}. Exiting.') + sys.exit() + + def load_records(self): + metadata_type = types.Metadata.FILE_RECORD + fp_csv = self.data_store.metadata(metadata_type) + self.log.info(f'loading: {fp_csv}') + if Path(fp_csv).is_file(): + self._metadata[metadata_type] = pd.read_csv(fp_csv).set_index('index') + else: + self.log.error(f'File not found: {fp_csv}. Exiting.') + sys.exit() + + def load_identities(self): + metadata_type = types.Metadata.IDENTITY + fp_csv = self.data_store.metadata(metadata_type) + self.log.info(f'loading: {fp_csv}') + if Path(fp_csv).is_file(): + self._metadata[metadata_type] = pd.read_csv(fp_csv).set_index('index') + else: + self.log.error(f'File not found: {fp_csv}. Exiting.') + sys.exit() def metadata(self, opt_metadata_type): - return self._metadata.get(opt_metadata_type, self._nullframe) + return self._metadata.get(opt_metadata_type, None) + + def index_to_record(self, index): + # get record meta + df_record = self._metadata[types.Metadata.FILE_RECORD] + ds_record = df_record.iloc[index] + identity_index = ds_record.identity_index + # get identity meta + df_identity = self._metadata[types.Metadata.IDENTITY] + # future datasets can have multiple identities per images + ds_identities = df_identity.iloc[identity_index] + # get filepath and S3 url + fp_im = self.data_store.face_image(ds_record.subdir, ds_record.fn, ds_record.ext) + s3_url = self.data_store_s3.face_image(ds_record.uuid) + image_record = ImageRecord(ds_record, fp_im, s3_url, ds_identities=ds_identities) + return image_record - def roi_idx_to_record(self, vector_index): + def vector_to_record(self, record_index): '''Accumulates image and its metadata''' df_face_vector = self._metadata[types.Metadata.FACE_VECTOR] ds_face_vector = df_face_vector.iloc[vector_index] @@ -115,18 +146,24 @@ class Dataset: for match_idx in match_idxs: # get the corresponding face vector row - self.log.debug(f'find match index: {match_idx}') - image_record = self.roi_idx_to_record(match_idx) + roi_index = self._face_vector_roi_idxs[match_idx] + self.log.debug(f'find match index: {match_idx}, --> roi_index: {roi_index}') + image_record = self.roi_idx_to_record(roi_index) image_records.append(image_record) return image_records # ---------------------------------------------------------------------- # utilities - def df_to_vec_list(self, df): + def df_vecs_to_dict(self, df): # convert the DataFrame CSV to float list of vecs return [list(map(float,x.vec.split(','))) for x in df.itertuples()] + def df_vec_idxs_to_dict(self, df): + # convert the DataFrame CSV to float list of vecs + #return [x.roi_index for x in df.itertuples()] + return [x.image_index for x in df.itertuples()] + def similar(self, query_vec, n_results): '''Finds most similar N indices of query face vector :query_vec: (list) of 128 floating point numbers of face encoding @@ -141,37 +178,35 @@ class Dataset: class ImageRecord: - def __init__(self, image_index, sha256, uuid, bbox, filepath, url): - self.image_index = image_index - self.sha256 = sha256 - self.uuid = uuid - self.bbox = bbox - self.filepath = filepath + def __init__(self, ds_record, fp, url, ds_rois=None, ds_identities=None): + # maybe more other meta will go there + self.image_index = ds_record.index + self.sha256 = ds_record.sha256 + self.uuid = ds_record.uuid + self.filepath = fp self.url = url - self._identity = None + self._identities = [] + # image records contain ROIs + # ROIs are linked to identities + + #self._identities = [Identity(x) for x in ds_identities] @property - def identity(self): + def identity(self, index): return self._identity - @identity.setter - def identity(self, value): - self._identity = value - def summarize(self): '''Summarizes data for debugging''' log = Logger.getLogger() log.info(f'filepath: {self.filepath}') log.info(f'sha256: {self.sha256}') log.info(f'UUID: {self.uuid}') - log.info(f'BBox: {self.bbox}') - log.info(f's3 url: {self.url}') - if self._identity: - log.info(f'name: {self._identity.name}') - log.info(f'age: {self._identity.age}') - log.info(f'gender: {self._identity.gender}') - log.info(f'nationality: {self._identity.nationality}') - log.info(f'images: {self._identity.n_images}') + log.info(f'S3 url: {self.url}') + for identity in self._identities: + log.info(f'fullname: {identity.fullname}') + log.info(f'description: {identity.description}') + log.info(f'gender: {identity.gender}') + log.info(f'images: {identity.n_images}') class Identity: diff --git a/megapixels/app/settings/app_cfg.py b/megapixels/app/settings/app_cfg.py index 7f9ed187..0c28b315 100644 --- a/megapixels/app/settings/app_cfg.py +++ b/megapixels/app/settings/app_cfg.py @@ -87,7 +87,7 @@ CKPT_ZERO_PADDING = 9 HASH_TREE_DEPTH = 3 HASH_BRANCH_SIZE = 3 -DLIB_FACEREC_JITTERS = 5 # number of face recognition jitters +DLIB_FACEREC_JITTERS = 25 # number of face recognition jitters DLIB_FACEREC_PADDING = 0.25 # default dlib POSE_MINMAX_YAW = (-25,25) diff --git a/megapixels/app/settings/types.py b/megapixels/app/settings/types.py index 685744aa..754be618 100644 --- a/megapixels/app/settings/types.py +++ b/megapixels/app/settings/types.py @@ -45,7 +45,7 @@ class LogLevel(Enum): # -------------------------------------------------------------------- class Metadata(Enum): - IDENTITY, FILEPATH, SHA256, UUID, FACE_VECTOR, FACE_POSE, FACE_ROI = range(7) + IDENTITY, FILE_RECORD, FACE_VECTOR, FACE_POSE, FACE_ROI = range(5) class Dataset(Enum): LFW, VGG_FACE2 = range(2) diff --git a/megapixels/app/utils/file_utils.py b/megapixels/app/utils/file_utils.py index 80239fe2..5c7b39d1 100644 --- a/megapixels/app/utils/file_utils.py +++ b/megapixels/app/utils/file_utils.py @@ -40,10 +40,16 @@ log = logging.getLogger(cfg.LOGGER_NAME) # File I/O read/write little helpers # ------------------------------------------ -def glob_multi(dir_in, exts): +def glob_multi(dir_in, exts, recursive=False): files = [] - for e in exts: - files.append(glob(join(dir_in, '*.{}'.format(e)))) + for ext in exts: + if recursive: + fp_glob = join(dir_in, '**/*.{}'.format(ext)) + log.info(f'glob {fp_glob}') + files += glob(fp_glob, recursive=True) + else: + fp_glob = join(dir_in, '*.{}'.format(ext)) + files += glob(fp_glob) return files diff --git a/megapixels/commands/cv/cluster.py b/megapixels/commands/cv/cluster.py index 94334133..419091a0 100644 --- a/megapixels/commands/cv/cluster.py +++ b/megapixels/commands/cv/cluster.py @@ -23,20 +23,20 @@ from app.utils.logger_utils import Logger @click.pass_context def cli(ctx, opt_data_store, opt_dataset, opt_metadata): """Display image info""" - - # cluster the embeddings -print("[INFO] clustering...") -clt = DBSCAN(metric="euclidean", n_jobs=args["jobs"]) -clt.fit(encodings) - -# determine the total number of unique faces found in the dataset -labelIDs = np.unique(clt.labels_) -numUniqueFaces = len(np.where(labelIDs > -1)[0]) -print("[INFO] # unique faces: {}".format(numUniqueFaces)) + + # cluster the embeddings + print("[INFO] clustering...") + clt = DBSCAN(metric="euclidean", n_jobs=args["jobs"]) + clt.fit(encodings) + + # determine the total number of unique faces found in the dataset + labelIDs = np.unique(clt.labels_) + numUniqueFaces = len(np.where(labelIDs > -1)[0]) + print("[INFO] # unique faces: {}".format(numUniqueFaces)) # load and display image im = cv.imread(fp_im) cv.imshow('', im) - + while True: k = cv.waitKey(1) & 0xFF if k == 27 or k == ord('q'): # ESC diff --git a/megapixels/commands/cv/face_pose.py b/megapixels/commands/cv/face_pose.py new file mode 100644 index 00000000..e7ffb7ac --- /dev/null +++ b/megapixels/commands/cv/face_pose.py @@ -0,0 +1,140 @@ +""" +Converts ROIs to pose: yaw, roll, pitch +""" + +import click + +from app.settings import types +from app.utils import click_utils +from app.settings import app_cfg as cfg + +@click.command() +@click.option('-i', '--input', 'opt_fp_in', default=None, + help='Override enum input filename CSV') +@click.option('-o', '--output', 'opt_fp_out', default=None, + help='Override enum output filename CSV') +@click.option('-m', '--media', 'opt_dir_media', default=None, + help='Override enum media directory') +@click.option('--data_store', 'opt_data_store', + type=cfg.DataStoreVar, + default=click_utils.get_default(types.DataStore.SSD), + show_default=True, + help=click_utils.show_help(types.Dataset)) +@click.option('--dataset', 'opt_dataset', + type=cfg.DatasetVar, + required=True, + show_default=True, + help=click_utils.show_help(types.Dataset)) +@click.option('--size', 'opt_size', + type=(int, int), default=(300, 300), + help='Output image size') +@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None), + help='Slice list of files') +@click.option('-f', '--force', 'opt_force', is_flag=True, + help='Force overwrite file') +@click.option('-d', '--display', 'opt_display', is_flag=True, + help='Display image for debugging') +@click.pass_context +def cli(ctx, opt_fp_in, opt_fp_out, opt_dir_media, opt_data_store, opt_dataset, opt_size, + opt_slice, opt_force, opt_display): + """Converts ROIs to pose: roll, yaw, pitch""" + + import sys + import os + from os.path import join + from pathlib import Path + from glob import glob + + from tqdm import tqdm + import numpy as np + import dlib # must keep a local reference for dlib + import cv2 as cv + import pandas as pd + + from app.models.bbox import BBox + from app.utils import logger_utils, file_utils, im_utils + from app.processors.face_landmarks import LandmarksDLIB + from app.processors.face_pose import FacePoseDLIB + from app.models.data_store import DataStore + + # ------------------------------------------------- + # init here + + log = logger_utils.Logger.getLogger() + + # set data_store + data_store = DataStore(opt_data_store, opt_dataset) + + # get filepath out + fp_out = data_store.metadata(types.Metadata.FACE_POSE) if opt_fp_out is None else opt_fp_out + if not opt_force and Path(fp_out).exists(): + log.error('File exists. Use "-f / --force" to overwite') + return + + # init face processors + face_pose = FacePoseDLIB() + face_landmarks = LandmarksDLIB() + + # load filepath data + fp_record = data_store.metadata(types.Metadata.FILE_RECORD) + df_record = pd.read_csv(fp_record).set_index('index') + # load ROI data + fp_roi = data_store.metadata(types.Metadata.FACE_ROI) + df_roi = pd.read_csv(fp_roi).set_index('index') + # slice if you want + if opt_slice: + df_roi = df_roi[opt_slice[0]:opt_slice[1]] + # group by image index (speedup if multiple faces per image) + df_img_groups = df_roi.groupby('record_index') + log.debug('processing {:,} groups'.format(len(df_img_groups))) + + # store poses and convert to DataFrame + poses = [] + + # iterate + for record_index, df_img_group in tqdm(df_img_groups): + # make fp + ds_record = df_record.iloc[record_index] + fp_im = data_store.face_image(ds_record.subdir, ds_record.fn, ds_record.ext) + im = cv.imread(fp_im) + # get bbox + x = df_img_group.x.values[0] + y = df_img_group.y.values[0] + w = df_img_group.w.values[0] + h = df_img_group.h.values[0] + dim = im.shape[:2][::-1] + bbox = BBox.from_xywh(x, y, w, h).to_dim(dim) + # get pose + landmarks = face_landmarks.landmarks(im, bbox) + pose_data = face_pose.pose(landmarks, dim, project_points=opt_display) + pose_degrees = pose_data['degrees'] # only keep the degrees data + + # use the project point data if display flag set + if opt_display: + pts_im = pose_data['points_image'] + pts_model = pose_data['points_model'] + pt_nose = pose_data['point_nose'] + dst = im.copy() + face_pose.draw_pose(dst, pts_im, pts_model, pt_nose) + face_pose.draw_degrees(dst, pose_degrees) + # display to cv window + cv.imshow('', dst) + while True: + k = cv.waitKey(1) & 0xFF + if k == 27 or k == ord('q'): # ESC + cv.destroyAllWindows() + sys.exit() + elif k != 255: + # any key to continue + break + + # add image index and append to result CSV data + pose_degrees['record_index'] = record_index + poses.append(pose_degrees) + + + # save date + file_utils.mkdirs(fp_out) + df = pd.DataFrame.from_dict(poses) + df.index.name = 'index' + df.to_csv(fp_out) \ No newline at end of file diff --git a/megapixels/commands/cv/face_roi.py b/megapixels/commands/cv/face_roi.py new file mode 100644 index 00000000..d7248aee --- /dev/null +++ b/megapixels/commands/cv/face_roi.py @@ -0,0 +1,171 @@ +""" +Crop images to prepare for training +""" + +import click +# from PIL import Image, ImageOps, ImageFilter, ImageDraw + +from app.settings import types +from app.utils import click_utils +from app.settings import app_cfg as cfg + +color_filters = {'color': 1, 'gray': 2, 'all': 3} + +@click.command() +@click.option('-i', '--input', 'opt_fp_in', default=None, + help='Override enum input filename CSV') +@click.option('-o', '--output', 'opt_fp_out', default=None, + help='Override enum output filename CSV') +@click.option('-m', '--media', 'opt_dir_media', default=None, + help='Override enum media directory') +@click.option('--data_store', 'opt_data_store', + type=cfg.DataStoreVar, + default=click_utils.get_default(types.DataStore.SSD), + show_default=True, + help=click_utils.show_help(types.Dataset)) +@click.option('--dataset', 'opt_dataset', + type=cfg.DatasetVar, + required=True, + show_default=True, + help=click_utils.show_help(types.Dataset)) +@click.option('--size', 'opt_size', + type=(int, int), default=(300, 300), + help='Output image size') +@click.option('-t', '--detector-type', 'opt_detector_type', + type=cfg.FaceDetectNetVar, + default=click_utils.get_default(types.FaceDetectNet.DLIB_CNN), + help=click_utils.show_help(types.FaceDetectNet)) +@click.option('-g', '--gpu', 'opt_gpu', default=0, + help='GPU index') +@click.option('--conf', 'opt_conf_thresh', default=0.85, type=click.FloatRange(0,1), + help='Confidence minimum threshold') +@click.option('-p', '--pyramids', 'opt_pyramids', default=0, type=click.IntRange(0,4), + help='Number pyramids to upscale for DLIB detectors') +@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None), + help='Slice list of files') +@click.option('--display/--no-display', 'opt_display', is_flag=True, default=False, + help='Display detections to debug') +@click.option('-f', '--force', 'opt_force', is_flag=True, + help='Force overwrite file') +@click.option('--color', 'opt_color_filter', + type=click.Choice(color_filters.keys()), default='all', + help='Filter to keep color or grayscale images (color = keep color') +@click.option('--largest/--all-faces', 'opt_largest', is_flag=True, default=True, + help='Only keep largest face') +@click.pass_context +def cli(ctx, opt_fp_in, opt_dir_media, opt_fp_out, opt_data_store, opt_dataset, opt_size, opt_detector_type, + opt_gpu, opt_conf_thresh, opt_pyramids, opt_slice, opt_display, opt_force, opt_color_filter, + opt_largest): + """Converts frames with faces to CSV of ROIs""" + + import sys + import os + from os.path import join + from pathlib import Path + from glob import glob + + from tqdm import tqdm + import numpy as np + import dlib # must keep a local reference for dlib + import cv2 as cv + import pandas as pd + + from app.utils import logger_utils, file_utils, im_utils + from app.processors import face_detector + from app.models.data_store import DataStore + + # ------------------------------------------------- + # init here + + log = logger_utils.Logger.getLogger() + + # set data_store + data_store = DataStore(opt_data_store, opt_dataset) + + # get filepath out + fp_out = data_store.metadata(types.Metadata.FACE_ROI) if opt_fp_out is None else opt_fp_out + if not opt_force and Path(fp_out).exists(): + log.error('File exists. Use "-f / --force" to overwite') + return + + # set detector + if opt_detector_type == types.FaceDetectNet.CVDNN: + detector = face_detector.DetectorCVDNN() + elif opt_detector_type == types.FaceDetectNet.DLIB_CNN: + detector = face_detector.DetectorDLIBCNN(opt_gpu) + elif opt_detector_type == types.FaceDetectNet.DLIB_HOG: + detector = face_detector.DetectorDLIBHOG() + elif opt_detector_type == types.FaceDetectNet.MTCNN: + detector = face_detector.DetectorMTCNN() + elif opt_detector_type == types.FaceDetectNet.HAAR: + log.error('{} not yet implemented'.format(opt_detector_type.name)) + return + + + # get list of files to process + fp_in = data_store.metadata(types.Metadata.FILE_RECORD) if opt_fp_in is None else opt_fp_in + df_records = pd.read_csv(fp_in).set_index('index') + if opt_slice: + df_records = df_records[opt_slice[0]:opt_slice[1]] + log.debug('processing {:,} files'.format(len(df_records))) + + # filter out grayscale + color_filter = color_filters[opt_color_filter] + + data = [] + + for df_record in tqdm(df_records.itertuples(), total=len(df_records)): + fp_im = data_store.face_image(str(df_record.subdir), str(df_record.fn), str(df_record.ext)) + im = cv.imread(fp_im) + + # filter out color or grayscale iamges + if color_filter != color_filters['all']: + try: + is_gray = im_utils.is_grayscale(im) + if is_gray and color_filter != color_filters['gray']: + log.debug('Skipping grayscale image: {}'.format(fp_im)) + continue + except Exception as e: + log.error('Could not check grayscale: {}'.format(fp_im)) + continue + + try: + bboxes = detector.detect(im, size=opt_size, pyramids=opt_pyramids, largest=opt_largest) + except Exception as e: + log.error('could not detect: {}'.format(fp_im)) + log.error('{}'.format(e)) + continue + + for bbox in bboxes: + roi = { + 'record_index': int(df_record.Index), + 'x': bbox.x, + 'y': bbox.y, + 'w': bbox.w, + 'h': bbox.h, + 'image_width': im.shape[1], + 'image_height': im.shape[0]} + data.append(roi) + + # debug display + if opt_display and len(bboxes): + bbox_dim = bbox.to_dim(im.shape[:2][::-1]) # w,h + im_md = im_utils.resize(im, width=min(1200, opt_size[0])) + for bbox in bboxes: + bbox_dim = bbox.to_dim(im_md.shape[:2][::-1]) + cv.rectangle(im_md, bbox_dim.pt_tl, bbox_dim.pt_br, (0,255,0), 3) + cv.imshow('', im_md) + while True: + k = cv.waitKey(1) & 0xFF + if k == 27 or k == ord('q'): # ESC + cv.destroyAllWindows() + sys.exit() + elif k != 255: + # any key to continue + break + + # save date + file_utils.mkdirs(fp_out) + df = pd.DataFrame.from_dict(data) + df.index.name = 'index' + df.to_csv(fp_out) \ No newline at end of file diff --git a/megapixels/commands/cv/face_vector.py b/megapixels/commands/cv/face_vector.py new file mode 100644 index 00000000..203f73eb --- /dev/null +++ b/megapixels/commands/cv/face_vector.py @@ -0,0 +1,125 @@ +""" +Converts ROIs to face vector +""" + +import click + +from app.settings import types +from app.utils import click_utils +from app.settings import app_cfg as cfg + +@click.command() +@click.option('-o', '--output', 'opt_fp_out', default=None, + help='Override enum output filename CSV') +@click.option('-m', '--media', 'opt_dir_media', default=None, + help='Override enum media directory') +@click.option('--data_store', 'opt_data_store', + type=cfg.DataStoreVar, + default=click_utils.get_default(types.DataStore.SSD), + show_default=True, + help=click_utils.show_help(types.Dataset)) +@click.option('--dataset', 'opt_dataset', + type=cfg.DatasetVar, + required=True, + show_default=True, + help=click_utils.show_help(types.Dataset)) +@click.option('--size', 'opt_size', + type=(int, int), default=(300, 300), + help='Output image size') +@click.option('-j', '--jitters', 'opt_jitters', default=cfg.DLIB_FACEREC_JITTERS, + help='Number of jitters') +@click.option('-p', '--padding', 'opt_padding', default=cfg.DLIB_FACEREC_PADDING, + help='Percentage padding') +@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None), + help='Slice list of files') +@click.option('-f', '--force', 'opt_force', is_flag=True, + help='Force overwrite file') +@click.option('-g', '--gpu', 'opt_gpu', default=0, + help='GPU index') +@click.pass_context +def cli(ctx, opt_fp_out, opt_dir_media, opt_data_store, opt_dataset, opt_size, + opt_slice, opt_force, opt_gpu, opt_jitters, opt_padding): + """Converts face ROIs to vectors""" + + import sys + import os + from os.path import join + from pathlib import Path + from glob import glob + + from tqdm import tqdm + import numpy as np + import dlib # must keep a local reference for dlib + import cv2 as cv + import pandas as pd + + from app.models.bbox import BBox + from app.models.data_store import DataStore + from app.utils import logger_utils, file_utils, im_utils + from app.processors import face_recognition + + + # ------------------------------------------------- + # init here + + log = logger_utils.Logger.getLogger() + # set data_store + data_store = DataStore(opt_data_store, opt_dataset) + + # get filepath out + fp_out = data_store.metadata(types.Metadata.FACE_VECTOR) if opt_fp_out is None else opt_fp_out + if not opt_force and Path(fp_out).exists(): + log.error('File exists. Use "-f / --force" to overwite') + return + + # init face processors + facerec = face_recognition.RecognitionDLIB() + + # load data + fp_record = data_store.metadata(types.Metadata.FILE_RECORD) + df_record = pd.read_csv(fp_record).set_index('index') + fp_roi = data_store.metadata(types.Metadata.FACE_ROI) + df_roi = pd.read_csv(fp_roi).set_index('index') + + if opt_slice: + df_roi = df_roi[opt_slice[0]:opt_slice[1]] + + # ------------------------------------------------- + # process here + df_img_groups = df_roi.groupby('record_index') + log.debug('processing {:,} groups'.format(len(df_img_groups))) + + vecs = [] + + for image_index, df_img_group in tqdm(df_img_groups): + # make fp + roi_index = df_img_group.index.values[0] + # log.debug(f'roi_index: {roi_index}, image_index: {image_index}') + ds_file = df_record.loc[roi_index] # locate image meta + #ds_file = df_record.loc['index', image_index] # locate image meta + + fp_im = data_store.face_image(str(ds_file.subdir), str(ds_file.fn), str(ds_file.ext)) + im = cv.imread(fp_im) + # get bbox + x = df_img_group.x.values[0] + y = df_img_group.y.values[0] + w = df_img_group.w.values[0] + h = df_img_group.h.values[0] + imw = df_img_group.image_width.values[0] + imh = df_img_group.image_height.values[0] + dim = im.shape[:2][::-1] + # get face vector + dim = (imw, imh) + bbox_dim = BBox.from_xywh(x, y, w, h).to_dim(dim) # convert to int real dimensions + # compute vec + # padding=opt_padding not yet implemented in 19.16 but merged in master + vec = facerec.vec(im, bbox_dim, jitters=opt_jitters) + vec_str = ','.join([repr(x) for x in vec]) # convert to string for CSV + vecs.append( {'roi_index': roi_index, 'image_index': image_index, 'vec': vec_str}) + + + # save date + df = pd.DataFrame.from_dict(vecs) + df.index.name = 'index' + file_utils.mkdirs(fp_out) + df.to_csv(fp_out) \ No newline at end of file diff --git a/megapixels/commands/cv/gen_face_vec.py b/megapixels/commands/cv/gen_face_vec.py deleted file mode 100644 index 83e1460d..00000000 --- a/megapixels/commands/cv/gen_face_vec.py +++ /dev/null @@ -1,123 +0,0 @@ -""" -Converts ROIs to face vector -""" - -import click - -from app.settings import types -from app.utils import click_utils -from app.settings import app_cfg as cfg - -@click.command() -@click.option('-o', '--output', 'opt_fp_out', default=None, - help='Override enum output filename CSV') -@click.option('-m', '--media', 'opt_dir_media', default=None, - help='Override enum media directory') -@click.option('--data_store', 'opt_data_store', - type=cfg.DataStoreVar, - default=click_utils.get_default(types.DataStore.SSD), - show_default=True, - help=click_utils.show_help(types.Dataset)) -@click.option('--dataset', 'opt_dataset', - type=cfg.DatasetVar, - required=True, - show_default=True, - help=click_utils.show_help(types.Dataset)) -@click.option('--size', 'opt_size', - type=(int, int), default=(300, 300), - help='Output image size') -@click.option('-j', '--jitters', 'opt_jitters', default=cfg.DLIB_FACEREC_JITTERS, - help='Number of jitters') -@click.option('-p', '--padding', 'opt_padding', default=cfg.DLIB_FACEREC_PADDING, - help='Percentage padding') -@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None), - help='Slice list of files') -@click.option('-f', '--force', 'opt_force', is_flag=True, - help='Force overwrite file') -@click.option('-g', '--gpu', 'opt_gpu', default=0, - help='GPU index') -@click.pass_context -def cli(ctx, opt_fp_out, opt_dir_media, opt_data_store, opt_dataset, opt_size, - opt_slice, opt_force, opt_gpu, opt_jitters, opt_padding): - """Converts face ROIs to vectors""" - - import sys - import os - from os.path import join - from pathlib import Path - from glob import glob - - from tqdm import tqdm - import numpy as np - import dlib # must keep a local reference for dlib - import cv2 as cv - import pandas as pd - - from app.models.bbox import BBox - from app.models.data_store import DataStore - from app.utils import logger_utils, file_utils, im_utils - from app.processors import face_recognition - - - # ------------------------------------------------- - # init here - - log = logger_utils.Logger.getLogger() - # set data_store - data_store = DataStore(opt_data_store, opt_dataset) - - # get filepath out - fp_out = data_store.metadata(types.Metadata.FACE_VECTOR) if opt_fp_out is None else opt_fp_out - if not opt_force and Path(fp_out).exists(): - log.error('File exists. Use "-f / --force" to overwite') - return - - # init face processors - facerec = face_recognition.RecognitionDLIB() - - # load data - df_file = pd.read_csv(data_store.metadata(types.Metadata.FILEPATH)).set_index('index') - df_roi = pd.read_csv(data_store.metadata(types.Metadata.FACE_ROI)).set_index('index') - - if opt_slice: - df_roi = df_roi[opt_slice[0]:opt_slice[1]] - - # ------------------------------------------------- - # process here - df_img_groups = df_roi.groupby('image_index') - log.debug('processing {:,} groups'.format(len(df_img_groups))) - - vecs = [] - - for image_index, df_img_group in tqdm(df_img_groups): - # make fp - roi_index = df_img_group.index.values[0] - log.debug(f'roi_index: {roi_index}, image_index: {image_index}') - ds_file = df_file.loc[roi_index] # locate image meta - #ds_file = df_file.loc['index', image_index] # locate image meta - - fp_im = data_store.face_image(str(ds_file.subdir), str(ds_file.fn), str(ds_file.ext)) - im = cv.imread(fp_im) - # get bbox - x = df_img_group.x.values[0] - y = df_img_group.y.values[0] - w = df_img_group.w.values[0] - h = df_img_group.h.values[0] - imw = df_img_group.image_width.values[0] - imh = df_img_group.image_height.values[0] - dim = im.shape[:2][::-1] - # get face vector - dim = (imw, imh) - bbox_dim = BBox.from_xywh(x, y, w, h).to_dim(dim) # convert to int real dimensions - # compute vec - # padding=opt_padding not yet implemented in 19.16 but merged in master - vec = facerec.vec(im, bbox_dim, jitters=opt_jitters) - vec_str = ','.join([repr(x) for x in vec]) # convert to string for CSV - vecs.append( {'roi_index': roi_index, 'image_index': image_index, 'vec': vec_str}) - - - # save date - df = pd.DataFrame.from_dict(vecs) - df.index.name = 'index' - #file_utils.mkdirs(fp_out) - #df.to_csv(fp_out) \ No newline at end of file diff --git a/megapixels/commands/cv/gen_pose.py b/megapixels/commands/cv/gen_pose.py deleted file mode 100644 index aefadb00..00000000 --- a/megapixels/commands/cv/gen_pose.py +++ /dev/null @@ -1,141 +0,0 @@ -""" -Converts ROIs to pose: yaw, roll, pitch -""" - -import click - -from app.settings import types -from app.utils import click_utils -from app.settings import app_cfg as cfg - -@click.command() -@click.option('-i', '--input', 'opt_fp_in', default=None, - help='Override enum input filename CSV') -@click.option('-o', '--output', 'opt_fp_out', default=None, - help='Override enum output filename CSV') -@click.option('-m', '--media', 'opt_dir_media', default=None, - help='Override enum media directory') -@click.option('--data_store', 'opt_data_store', - type=cfg.DataStoreVar, - default=click_utils.get_default(types.DataStore.SSD), - show_default=True, - help=click_utils.show_help(types.Dataset)) -@click.option('--dataset', 'opt_dataset', - type=cfg.DatasetVar, - required=True, - show_default=True, - help=click_utils.show_help(types.Dataset)) -@click.option('--size', 'opt_size', - type=(int, int), default=(300, 300), - help='Output image size') -@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None), - help='Slice list of files') -@click.option('-f', '--force', 'opt_force', is_flag=True, - help='Force overwrite file') -@click.option('-d', '--display', 'opt_display', is_flag=True, - help='Display image for debugging') -@click.pass_context -def cli(ctx, opt_fp_in, opt_fp_out, opt_dir_media, opt_data_store, opt_dataset, opt_size, - opt_slice, opt_force, opt_display): - """Converts ROIs to pose: roll, yaw, pitch""" - - import sys - import os - from os.path import join - from pathlib import Path - from glob import glob - - from tqdm import tqdm - import numpy as np - import dlib # must keep a local reference for dlib - import cv2 as cv - import pandas as pd - - from app.models.bbox import BBox - from app.utils import logger_utils, file_utils, im_utils - from app.processors.face_landmarks import LandmarksDLIB - from app.processors.face_pose import FacePoseDLIB - from app.models.data_store import DataStore - - # ------------------------------------------------- - # init here - - log = logger_utils.Logger.getLogger() - - # set data_store - data_store = DataStore(opt_data_store, opt_dataset) - - # get filepath out - fp_out = data_store.metadata(types.Metadata.FACE_POSE) if opt_fp_out is None else opt_fp_out - if not opt_force and Path(fp_out).exists(): - log.error('File exists. Use "-f / --force" to overwite') - return - - # init face processors - face_pose = FacePoseDLIB() - face_landmarks = LandmarksDLIB() - - # load filepath data - fp_filepath = data_store.metadata(types.Metadata.FILEPATH) - df_filepath = pd.read_csv(fp_filepath) - # load ROI data - fp_roi = data_store.metadata(types.Metadata.FACE_ROI) - df_roi = pd.read_csv(fp_roi) - # slice if you want - if opt_slice: - df_roi = df_roi[opt_slice[0]:opt_slice[1]] - # group by image index (speedup if multiple faces per image) - df_img_groups = df_roi.groupby('image_index') - log.debug('processing {:,} groups'.format(len(df_img_groups))) - - # store poses and convert to DataFrame - poses = [] - - # iterate - for image_index, df_img_group in tqdm(df_img_groups): - # make fp - ds_file = df_filepath.iloc[image_index] - fp_im = data_store.face_image(ds_file.subdir, ds_file.fn, ds_file.ext) - #fp_im = join(opt_dir_media, ds_file.subdir, '{}.{}'.format(ds_file.fn, ds_file.ext)) - im = cv.imread(fp_im) - # get bbox - x = df_img_group.x.values[0] - y = df_img_group.y.values[0] - w = df_img_group.w.values[0] - h = df_img_group.h.values[0] - dim = im.shape[:2][::-1] - bbox = BBox.from_xywh(x, y, w, h).to_dim(dim) - # get pose - landmarks = face_landmarks.landmarks(im, bbox) - pose_data = face_pose.pose(landmarks, dim, project_points=opt_display) - pose_degrees = pose_data['degrees'] # only keep the degrees data - - # use the project point data if display flag set - if opt_display: - pts_im = pose_data['points_image'] - pts_model = pose_data['points_model'] - pt_nose = pose_data['point_nose'] - dst = im.copy() - face_pose.draw_pose(dst, pts_im, pts_model, pt_nose) - face_pose.draw_degrees(dst, pose_degrees) - # display to cv window - cv.imshow('', dst) - while True: - k = cv.waitKey(1) & 0xFF - if k == 27 or k == ord('q'): # ESC - cv.destroyAllWindows() - sys.exit() - elif k != 255: - # any key to continue - break - - # add image index and append to result CSV data - pose_degrees['image_index'] = image_index - poses.append(pose_degrees) - - - # save date - file_utils.mkdirs(fp_out) - df = pd.DataFrame.from_dict(poses) - df.index.name = 'index' - df.to_csv(fp_out) \ No newline at end of file diff --git a/megapixels/commands/cv/gen_rois.py b/megapixels/commands/cv/gen_rois.py deleted file mode 100644 index 20dd598a..00000000 --- a/megapixels/commands/cv/gen_rois.py +++ /dev/null @@ -1,172 +0,0 @@ -""" -Crop images to prepare for training -""" - -import click -# from PIL import Image, ImageOps, ImageFilter, ImageDraw - -from app.settings import types -from app.utils import click_utils -from app.settings import app_cfg as cfg - -color_filters = {'color': 1, 'gray': 2, 'all': 3} - -@click.command() -@click.option('-i', '--input', 'opt_fp_in', default=None, - help='Override enum input filename CSV') -@click.option('-o', '--output', 'opt_fp_out', default=None, - help='Override enum output filename CSV') -@click.option('-m', '--media', 'opt_dir_media', default=None, - help='Override enum media directory') -@click.option('--data_store', 'opt_data_store', - type=cfg.DataStoreVar, - default=click_utils.get_default(types.DataStore.SSD), - show_default=True, - help=click_utils.show_help(types.Dataset)) -@click.option('--dataset', 'opt_dataset', - type=cfg.DatasetVar, - required=True, - show_default=True, - help=click_utils.show_help(types.Dataset)) -@click.option('--size', 'opt_size', - type=(int, int), default=(300, 300), - help='Output image size') -@click.option('-t', '--detector-type', 'opt_detector_type', - type=cfg.FaceDetectNetVar, - default=click_utils.get_default(types.FaceDetectNet.DLIB_CNN), - help=click_utils.show_help(types.FaceDetectNet)) -@click.option('-g', '--gpu', 'opt_gpu', default=0, - help='GPU index') -@click.option('--conf', 'opt_conf_thresh', default=0.85, type=click.FloatRange(0,1), - help='Confidence minimum threshold') -@click.option('-p', '--pyramids', 'opt_pyramids', default=0, type=click.IntRange(0,4), - help='Number pyramids to upscale for DLIB detectors') -@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None), - help='Slice list of files') -@click.option('--display/--no-display', 'opt_display', is_flag=True, default=False, - help='Display detections to debug') -@click.option('-f', '--force', 'opt_force', is_flag=True, - help='Force overwrite file') -@click.option('--color', 'opt_color_filter', - type=click.Choice(color_filters.keys()), default='all', - help='Filter to keep color or grayscale images (color = keep color') -@click.option('--largest/--all-faces', 'opt_largest', is_flag=True, default=True, - help='Only keep largest face') -@click.pass_context -def cli(ctx, opt_fp_in, opt_dir_media, opt_fp_out, opt_data_store, opt_dataset, opt_size, opt_detector_type, - opt_gpu, opt_conf_thresh, opt_pyramids, opt_slice, opt_display, opt_force, opt_color_filter, - opt_largest): - """Converts frames with faces to CSV of ROIs""" - - import sys - import os - from os.path import join - from pathlib import Path - from glob import glob - - from tqdm import tqdm - import numpy as np - import dlib # must keep a local reference for dlib - import cv2 as cv - import pandas as pd - - from app.utils import logger_utils, file_utils, im_utils - from app.processors import face_detector - from app.models.data_store import DataStore - - # ------------------------------------------------- - # init here - - log = logger_utils.Logger.getLogger() - - # set data_store - data_store = DataStore(opt_data_store, opt_dataset) - - # get filepath out - fp_out = data_store.metadata(types.Metadata.FACE_ROI) if opt_fp_out is None else opt_fp_out - if not opt_force and Path(fp_out).exists(): - log.error('File exists. Use "-f / --force" to overwite') - return - - # set detector - if opt_detector_type == types.FaceDetectNet.CVDNN: - detector = face_detector.DetectorCVDNN() - elif opt_detector_type == types.FaceDetectNet.DLIB_CNN: - detector = face_detector.DetectorDLIBCNN(opt_gpu) - elif opt_detector_type == types.FaceDetectNet.DLIB_HOG: - detector = face_detector.DetectorDLIBHOG() - elif opt_detector_type == types.FaceDetectNet.MTCNN: - detector = face_detector.DetectorMTCNN() - elif opt_detector_type == types.FaceDetectNet.HAAR: - log.error('{} not yet implemented'.format(opt_detector_type.name)) - return - - - # get list of files to process - fp_in = data_store.metadata(types.Metadata.FILEPATH) if opt_fp_in is None else opt_fp_in - df_files = pd.read_csv(fp_in).set_index('index') - if opt_slice: - df_files = df_files[opt_slice[0]:opt_slice[1]] - log.debug('processing {:,} files'.format(len(df_files))) - - # filter out grayscale - color_filter = color_filters[opt_color_filter] - - data = [] - - for df_file in tqdm(df_files.itertuples(), total=len(df_files)): - fp_im = data_store.face_image(str(df_file.subdir), str(df_file.fn), str(df_file.ext)) - #fp_im = join(opt_dir_media, str(df_file.subdir), f'{df_file.fn}.{df_file.ext}') - im = cv.imread(fp_im) - - # filter out color or grayscale iamges - if color_filter != color_filters['all']: - try: - is_gray = im_utils.is_grayscale(im) - if is_gray and color_filter != color_filters['gray']: - log.debug('Skipping grayscale image: {}'.format(fp_im)) - continue - except Exception as e: - log.error('Could not check grayscale: {}'.format(fp_im)) - continue - - try: - bboxes = detector.detect(im, size=opt_size, pyramids=opt_pyramids, largest=opt_largest) - except Exception as e: - log.error('could not detect: {}'.format(fp_im)) - log.error('{}'.format(e)) - continue - - for bbox in bboxes: - roi = { - 'image_index': int(df_file.Index), - 'x': bbox.x, - 'y': bbox.y, - 'w': bbox.w, - 'h': bbox.h, - 'image_width': im.shape[1], - 'image_height': im.shape[0]} - data.append(roi) - - # debug display - if opt_display and len(bboxes): - bbox_dim = bbox.to_dim(im.shape[:2][::-1]) # w,h - im_md = im_utils.resize(im, width=min(1200, opt_size[0])) - for bbox in bboxes: - bbox_dim = bbox.to_dim(im_md.shape[:2][::-1]) - cv.rectangle(im_md, bbox_dim.pt_tl, bbox_dim.pt_br, (0,255,0), 3) - cv.imshow('', im_md) - while True: - k = cv.waitKey(1) & 0xFF - if k == 27 or k == ord('q'): # ESC - cv.destroyAllWindows() - sys.exit() - elif k != 255: - # any key to continue - break - - # save date - file_utils.mkdirs(fp_out) - df = pd.DataFrame.from_dict(data) - df.index.name = 'index' - df.to_csv(opt_fp_out) \ No newline at end of file diff --git a/megapixels/commands/datasets/filter_by_pose.py b/megapixels/commands/datasets/filter_by_pose.py index 6fdbef98..a588b18e 100644 --- a/megapixels/commands/datasets/filter_by_pose.py +++ b/megapixels/commands/datasets/filter_by_pose.py @@ -53,17 +53,11 @@ def cli(ctx, opt_fp_in, opt_fp_out, opt_data_store, opt_dataset, opt_yaw, opt_ro fp_roi = data_store.metadata(types.Metadata.FACE_ROI) df_roi = pd.read_csv(fp_roi).set_index('index') # load filepath - fp_filepath = data_store.metadata(types.Metadata.FILEPATH) - df_filepath = pd.read_csv(fp_filepath).set_index('index') - # load uuid - fp_uuid= data_store.metadata(types.Metadata.UUID) - df_uuid = pd.read_csv(fp_uuid).set_index('index') - # load sha256 index - fp_sha256 = data_store.metadata(types.Metadata.SHA256) - df_sha256 = pd.read_csv(fp_sha256).set_index('index') + fp_record = data_store.metadata(types.Metadata.FILE_RECORD) + df_record = pd.read_csv(fp_record).set_index('index') # debug log.info('Processing {:,} rows'.format(len(df_pose))) - n_rows = len(df_pose) + n_rows = len(df_record) # filter out extreme poses invalid_indices = [] @@ -74,28 +68,29 @@ def cli(ctx, opt_fp_in, opt_fp_out, opt_data_store, opt_dataset, opt_yaw, opt_ro invalid_indices.append(ds_pose.Index) # unique file indexs # filter out valid/invalid - log.info(invalid_indices[:20]) + log.info(f'indices 0-20: {invalid_indices[:20]}') log.info(f'Removing {len(invalid_indices)} invalid indices...') - df_filepath = df_filepath.drop(df_pose.index[invalid_indices]) - df_sha256 = df_sha256.drop(df_pose.index[invalid_indices]) - df_uuid = df_uuid.drop(df_pose.index[invalid_indices]) - df_roi = df_roi.drop(df_pose.index[invalid_indices]) + df_record = df_record.drop(df_record.index[invalid_indices]) + df_roi = df_roi.drop(df_roi.index[invalid_indices]) df_pose = df_pose.drop(df_pose.index[invalid_indices]) - log.info(f'Removed {n_rows - len(df_pose)}') + log.info(f'Removed {n_rows - len(df_record)}') # move file to make backup dir_bkup = join(Path(fp_pose).parent, f'backup_{datetime.now():%Y%m%d_%M%S}') file_utils.mkdirs(dir_bkup) # move files to backup - shutil.move(fp_filepath, join(dir_bkup, Path(fp_filepath).name)) - shutil.move(fp_sha256, join(dir_bkup, Path(fp_sha256).name)) - shutil.move(fp_uuid, join(dir_bkup, Path(fp_uuid).name)) + shutil.move(fp_record, join(dir_bkup, Path(fp_record).name)) shutil.move(fp_roi, join(dir_bkup, Path(fp_roi).name)) shutil.move(fp_pose, join(dir_bkup, Path(fp_pose).name)) - # save filtered poses - df_filepath.to_csv(fp_filepath) - df_sha256.to_csv(fp_sha256) - df_uuid.to_csv(fp_uuid) + # resave file records + df_record = df_record.reset_index(drop=True) + df_record.index.name = 'index' + df_record.to_csv(fp_record) + # resave ROI + df_roi = df_roi.reset_index(drop=True) + df_roi.index.name = 'index' df_roi.to_csv(fp_roi) + # resave pose + df_pose = df_pose.reset_index(drop=True) + df_pose.index.name = 'index' df_pose.to_csv(fp_pose) - diff --git a/megapixels/commands/datasets/gen_filepath.py b/megapixels/commands/datasets/gen_filepath.py index e06fee6b..5db405c0 100644 --- a/megapixels/commands/datasets/gen_filepath.py +++ b/megapixels/commands/datasets/gen_filepath.py @@ -50,7 +50,7 @@ def cli(ctx, opt_fp_in, opt_fp_out, opt_data_store, opt_dataset, opt_slice, from tqdm import tqdm from glob import glob - from app.models import DataStore + from app.models.data_store import DataStore from app.utils import file_utils, im_utils data_store = DataStore(opt_data_store, opt_dataset) @@ -97,6 +97,6 @@ def cli(ctx, opt_fp_in, opt_fp_out, opt_data_store, opt_dataset, opt_slice, file_utils.mkdirs(fp_out) df_filepath = pd.DataFrame.from_dict(data) df_filepath = df_filepath.sort_values(by=['subdir'], ascending=True) - df_filepath = df_filepath.reset_index(drop=True) + df_filepath = df_filepath.reset_index() df_filepath.index.name = 'index' df_filepath.to_csv(fp_out) \ No newline at end of file diff --git a/megapixels/commands/datasets/gen_sha256.py b/megapixels/commands/datasets/gen_sha256.py deleted file mode 100644 index 1616eebf..00000000 --- a/megapixels/commands/datasets/gen_sha256.py +++ /dev/null @@ -1,152 +0,0 @@ -''' - -''' -import click - -from app.settings import types -from app.utils import click_utils -from app.settings import app_cfg as cfg -from app.utils.logger_utils import Logger - -log = Logger.getLogger() - -identity_sources = ['subdir', 'subdir_head', 'subdir_tail'] - -@click.command() -@click.option('-i', '--input', 'opt_fp_in', default=None, - help='Override enum input filename CSV') -@click.option('-o', '--output', 'opt_fp_out', default=None, - help='Override enum output filename CSV') -@click.option('-m', '--media', 'opt_dir_media', default=None, - help='Override enum media directory') -@click.option('--data_store', 'opt_data_store', - type=cfg.DataStoreVar, - default=click_utils.get_default(types.DataStore.NAS), - show_default=True, - help=click_utils.show_help(types.Dataset)) -@click.option('--dataset', 'opt_dataset', - type=cfg.DatasetVar, - required=True, - show_default=True, - help=click_utils.show_help(types.Dataset)) -@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None), - help='Slice list of files') -@click.option('-t', '--threads', 'opt_threads', default=12, - help='Number of threads') -@click.option('-f', '--force', 'opt_force', is_flag=True, - help='Force overwrite file') -@click.option('--identity', 'opt_identity', default='subdir_tail', type=click.Choice(identity_sources), - help='Identity source, blank for no identity') -@click.pass_context -def cli(ctx, opt_fp_in, opt_fp_out, opt_dataset, opt_data_store, opt_dir_media, opt_slice, opt_threads, - opt_identity, opt_force): - """Generates sha256/identity index CSV file""" - - import sys - from glob import glob - from os.path import join - from pathlib import Path - import time - from multiprocessing.dummy import Pool as ThreadPool - import random - - import pandas as pd - from tqdm import tqdm - from glob import glob - - from app.models import DataStore - from app.utils import file_utils, im_utils - - - # set data_store - data_store = DataStore(opt_data_store, opt_dataset) - # get filepath out - fp_out = data_store.metadata(types.Metadata.SHA256) if opt_fp_out is None else opt_fp_out - # exit if exists - if not opt_force and Path(fp_out).exists(): - log.error('File exists. Use "-f / --force" to overwite') - return - # get filepath in - fp_in = data_store.metadata(types.Metadata.FILEPATH) - df_files = pd.read_csv(fp_in).set_index('index') - # slice if you want - if opt_slice: - df_files = df_files[opt_slice[0]:opt_slice[1]] - - log.info('Processing {:,} images'.format(len(df_files))) - - - # prepare list of images to multithread into sha256s - dir_media = data_store.media_images_original() if opt_dir_media is None else opt_dir_media - file_objs = [] - for ds_file in df_files.itertuples(): - fp_im = join(dir_media, str(ds_file.subdir), f"{ds_file.fn}.{ds_file.ext}") - # find the image_index - # append the subdir option, sort by this then increment by unique subdir - file_obj = {'fp': fp_im, 'index': ds_file.Index} - if opt_identity: - subdirs = ds_file.subdir.split('/') - if not len(subdirs) > 0: - log.error(f'Could not split subdir: "{ds_file.subdir}. Try different option for "--identity"') - log.error('exiting') - return - if opt_identity == 'subdir': - subdir = subdirs[0] - elif opt_identity == 'subdir_head': - # use first part of subdir path - subdir = subdirs[0] - elif opt_identity == 'subdir_tail': - # use last part of subdir path - subdir = subdirs[-1] - file_obj['identity_subdir'] = subdir - file_objs.append(file_obj) - - # convert to thread pool - pbar = tqdm(total=len(file_objs)) - - def as_sha256(file_obj): - pbar.update(1) - file_obj['sha256'] = file_utils.sha256(file_obj['fp']) - return file_obj - - # multithread pool - pool_file_objs = [] - st = time.time() - pool = ThreadPool(opt_threads) - with tqdm(total=len(file_objs)) as pbar: - pool_file_objs = pool.map(as_sha256, file_objs) - pbar.close() - - # convert data to dict - data = [] - for pool_file_obj in pool_file_objs: - data.append( { - 'sha256': pool_file_obj['sha256'], - 'index': pool_file_obj['index'], - 'identity_subdir': pool_file_obj.get('identity_subdir', ''), - }) - - # sort based on identity_subdir - # save to CSV - df_sha256 = pd.DataFrame.from_dict(data) - # add new column for identity - df_sha256['identity_index'] = [1] * len(df_sha256) - df_sha256 = df_sha256.sort_values(by=['identity_subdir'], ascending=True) - df_sha256_identity_groups = df_sha256.groupby('identity_subdir') - for identity_index, df_sha256_identity_group_tuple in enumerate(df_sha256_identity_groups): - identity_subdir, df_sha256_identity_group = df_sha256_identity_group_tuple - for ds_sha256 in df_sha256_identity_group.itertuples(): - df_sha256.at[ds_sha256.Index, 'identity_index'] = identity_index - # drop temp identity subdir column - df_sha256 = df_sha256.drop('identity_subdir', axis=1) - # write to CSV - log.info(f'rows: {len(df_sha256)}') - file_utils.mkdirs(fp_out) - df_sha256.set_index('index') - df_sha256 = df_sha256.sort_values(['index'], ascending=[True]) - df_sha256.to_csv(fp_out, index=False) - - # timing - log.info(f'wrote file: {fp_out}') - log.info('time: {:.2f}, theads: {}'.format(time.time() - st, opt_threads)) - \ No newline at end of file diff --git a/megapixels/commands/datasets/gen_uuid.py b/megapixels/commands/datasets/gen_uuid.py index 612c43ee..d7e7b52c 100644 --- a/megapixels/commands/datasets/gen_uuid.py +++ b/megapixels/commands/datasets/gen_uuid.py @@ -37,7 +37,7 @@ def cli(ctx, opt_fp_in, opt_fp_out, opt_data_store, opt_dataset, opt_force): from tqdm import tqdm import pandas as pd - from app.models import DataStore + from app.models.data_store import DataStore # set data_store diff --git a/megapixels/commands/datasets/identity_meta_lfw.py b/megapixels/commands/datasets/identity_meta_lfw.py new file mode 100644 index 00000000..45386b23 --- /dev/null +++ b/megapixels/commands/datasets/identity_meta_lfw.py @@ -0,0 +1,93 @@ +''' +add identity from description using subdir +''' +import click + +from app.settings import types +from app.models.dataset import Dataset +from app.utils import click_utils +from app.settings import app_cfg as cfg +from app.utils.logger_utils import Logger + +log = Logger.getLogger() + +@click.command() +@click.option('-i', '--input', 'opt_fp_in', required=True, + help='Identity meta file') +@click.option('-o', '--output', 'opt_fp_out', default=None, + help='Override enum output filename CSV') +@click.option('--column', 'opt_identity_key', default='identity_key', + help='Match column') +@click.option('--data_store', 'opt_data_store', + type=cfg.DataStoreVar, + default=click_utils.get_default(types.DataStore.SSD), + show_default=True, + help=click_utils.show_help(types.Dataset)) +@click.option('-f', '--force', 'opt_force', is_flag=True, + help='Force overwrite file') +@click.pass_context +def cli(ctx, opt_fp_in, opt_fp_out, opt_identity_key, opt_data_store, opt_force): + """Display image info""" + + import sys + from glob import glob + from os.path import join + from pathlib import Path + import time + + import pandas as pd + import cv2 as cv + from tqdm import tqdm + + from app.utils import file_utils, im_utils + from app.models.data_store import DataStore + + log = Logger.getLogger() + + # output file + opt_dataset = types.Dataset.LFW + data_store = DataStore(opt_data_store, opt_dataset) + fp_out = data_store.metadata(types.Metadata.IDENTITY) if opt_fp_out is None else opt_fp_out + # exit if exists + log.debug(fp_out) + if not opt_force and Path(fp_out).exists(): + log.error('File exists. Use "-f / --force" to overwite') + return + + # init dataset + # load file records + fp_record = data_store.metadata(types.Metadata.FILE_RECORD) + df_record = pd.read_csv(fp_record).set_index('index') + + # load identity meta + # this file is maybe prepared in a Jupyter notebook + # the "identity_key" + df_identity_meta = pd.read_csv(opt_fp_in).set_index('index') + # create a new file called 'identity.csv' + identities = [] + # iterate records and get identity index where 'identity_key' matches + log.debug(type(df_record)) + identity_indices = [] + for record_idx, ds_record in tqdm(df_record.iterrows(), total=len(df_record)): + identity_value = ds_record[opt_identity_key] + identity_index = ds_record.identity_index + ds_identity_meta = df_identity_meta.loc[(df_identity_meta[opt_identity_key] == identity_value)] + if identity_index not in identity_indices: + identity_indices.append(identity_index) + identities.append({ + 'description': ds_identity_meta.description.values[0], + 'name': ds_identity_meta.name.values[0], + 'images': ds_identity_meta.images.values[0], + 'gender': ds_identity_meta.gender.values[0], + }) + + # write to csv + df_identity = pd.DataFrame.from_dict(identities) + df_identity.index.name = 'index' + df_identity.to_csv(fp_out) + ''' + index,name,name_orig,description,gender,images,image_index,identity_key + 0,A. J. Cook,AJ Cook,Canadian actress,f,1,0,AJ_Cook + ''' + + diff --git a/megapixels/commands/datasets/identity_meta_vgg_face2.py b/megapixels/commands/datasets/identity_meta_vgg_face2.py new file mode 100644 index 00000000..85b6644d --- /dev/null +++ b/megapixels/commands/datasets/identity_meta_vgg_face2.py @@ -0,0 +1,88 @@ +''' +add identity from description using subdir +''' +import click + +from app.settings import types +from app.models.dataset import Dataset +from app.utils import click_utils +from app.settings import app_cfg as cfg +from app.utils.logger_utils import Logger + +log = Logger.getLogger() + +@click.command() +@click.option('-i', '--input', 'opt_fp_in', required=True, + help='Identity meta file') +@click.option('-o', '--output', 'opt_fp_out', default=None, + help='Override enum output filename CSV') +@click.option('--data_store', 'opt_data_store', + type=cfg.DataStoreVar, + default=click_utils.get_default(types.DataStore.SSD), + show_default=True, + help=click_utils.show_help(types.Dataset)) +@click.option('-f', '--force', 'opt_force', is_flag=True, + help='Force overwrite file') +@click.pass_context +def cli(ctx, opt_fp_in, opt_fp_out, opt_data_store, opt_force): + """Display image info""" + + import sys + from glob import glob + from os.path import join + from pathlib import Path + import time + + import pandas as pd + import cv2 as cv + from tqdm import tqdm + + from app.utils import file_utils, im_utils + from app.models.data_store import DataStore + + log = Logger.getLogger() + + # output file + opt_dataset = types.Dataset.VGG_FACE2 + data_store = DataStore(opt_data_store, opt_dataset) + fp_out = data_store.metadata(types.Metadata.IDENTITY) if opt_fp_out is None else opt_fp_out + # exit if exists + log.debug(fp_out) + if not opt_force and Path(fp_out).exists(): + log.error('File exists. Use "-f / --force" to overwite') + return + + # init dataset + # load file records + identity_key = 'identity_key' + fp_record = data_store.metadata(types.Metadata.FILE_RECORD) + df_record = pd.read_csv(fp_record).set_index('index') + + # load identity meta + # this file is maybe prepared in a Jupyter notebook + # the "identity_key" + df_identity_meta = pd.read_csv(opt_fp_in).set_index('index') + # create a new file called 'identity.csv' + identities = [] + # iterate records and get identity index where 'identity_key' matches + log.debug(type(df_record)) + identity_indices = [] + for ds_record in tqdm(df_record.itertuples(), total=len(df_record)): + identity_value = ds_record.identity_key + identity_index = ds_record.identity_index + ds_identity_meta = df_identity_meta.loc[(df_identity_meta[identity_key] == identity_value)] + if identity_index not in identity_indices: + identity_indices.append(identity_index) + identities.append({ + 'description': ds_identity_meta.description.values[0], + 'name': ds_identity_meta.name.values[0], + 'images': ds_identity_meta.images.values[0], + 'gender': ds_identity_meta.gender.values[0], + }) + + # write to csv + df_identity = pd.DataFrame.from_dict(identities) + df_identity.index.name = 'index' + df_identity.to_csv(fp_out) + + diff --git a/megapixels/commands/datasets/lookup.py b/megapixels/commands/datasets/lookup.py index 5a2a171e..c1c66c19 100644 --- a/megapixels/commands/datasets/lookup.py +++ b/megapixels/commands/datasets/lookup.py @@ -13,7 +13,7 @@ log = Logger.getLogger() help='Vector index to lookup') @click.option('--data_store', 'opt_data_store', type=cfg.DataStoreVar, - default=click_utils.get_default(types.DataStore.NAS), + default=click_utils.get_default(types.DataStore.SSD), show_default=True, help=click_utils.show_help(types.Dataset)) @click.option('--dataset', 'opt_dataset', @@ -41,11 +41,12 @@ def cli(ctx, opt_index, opt_data_store, opt_dataset): log = Logger.getLogger() # init dataset dataset = Dataset(opt_data_store, opt_dataset) + #dataset.load_face_vectors() + dataset.load_records() + dataset.load_identities() # set data store and load files - dataset.load() # find image records - image_record = dataset.roi_idx_to_record(opt_index) - # debug + image_record = dataset.index_to_record(opt_index) image_record.summarize() # load image im = cv.imread(image_record.filepath) diff --git a/megapixels/commands/datasets/records.py b/megapixels/commands/datasets/records.py new file mode 100644 index 00000000..80de5040 --- /dev/null +++ b/megapixels/commands/datasets/records.py @@ -0,0 +1,159 @@ +''' + +''' +import click + +from app.settings import types +from app.utils import click_utils +from app.settings import app_cfg as cfg +from app.utils.logger_utils import Logger + +log = Logger.getLogger() + +identity_sources = ['subdir', 'subdir_head', 'subdir_tail'] + +@click.command() +@click.option('-i', '--input', 'opt_fp_in', default=None, + help='Override enum input filename CSV') +@click.option('-o', '--output', 'opt_fp_out', default=None, + help='Override enum output filename CSV') +@click.option('-m', '--media', 'opt_dir_media', default=None, + help='Override enum media directory') +@click.option('--data_store', 'opt_data_store', + type=cfg.DataStoreVar, + default=click_utils.get_default(types.DataStore.SSD), + show_default=True, + help=click_utils.show_help(types.Dataset)) +@click.option('--dataset', 'opt_dataset', + type=cfg.DatasetVar, + required=True, + show_default=True, + help=click_utils.show_help(types.Dataset)) +@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None), + help='Slice list of files') +@click.option('-t', '--threads', 'opt_threads', default=12, + help='Number of threads') +@click.option('-f', '--force', 'opt_force', is_flag=True, + help='Force overwrite file') +@click.option('--identity', 'opt_identity', default=None, type=click.Choice(identity_sources), + help='Identity source, blank for no identity') +@click.option('--recursive/--no-recursive', 'opt_recursive', is_flag=True, default=False, + help='Use glob recursion (slower)') +@click.pass_context +def cli(ctx, opt_fp_in, opt_fp_out, opt_dataset, opt_data_store, opt_dir_media, opt_slice, opt_threads, + opt_identity, opt_force, opt_recursive): + """Generates sha256, uuid, and identity index CSV file""" + + import sys + from glob import glob + from os.path import join + from pathlib import Path + import time + from multiprocessing.dummy import Pool as ThreadPool + import random + import uuid + + import pandas as pd + from tqdm import tqdm + from glob import glob + + from app.models.data_store import DataStore + from app.utils import file_utils, im_utils + + + # set data_store + data_store = DataStore(opt_data_store, opt_dataset) + # get filepath out + fp_out = data_store.metadata(types.Metadata.FILE_RECORD) if opt_fp_out is None else opt_fp_out + # exit if exists + if not opt_force and Path(fp_out).exists(): + log.error('File exists. Use "-f / --force" to overwite') + return + + # ---------------------------------------------------------------- + # glob files + + fp_in = opt_fp_in if opt_fp_in is not None else data_store.media_images_original() + log.info(f'Globbing {fp_in}') + fp_ims = file_utils.glob_multi(fp_in, ['jpg', 'png'], recursive=opt_recursive) + # fail if none + if not fp_ims: + log.error('No images. Try with "--recursive"') + return + # slice to reduce + if opt_slice: + fp_ims = fp_ims[opt_slice[0]:opt_slice[1]] + log.info('Found {:,} images'.format(len(fp_ims))) + + + # ---------------------------------------------------------------- + # multithread process into SHA256 + + pbar = tqdm(total=len(fp_ims)) + + def as_sha256(fp_im): + pbar.update(1) + return file_utils.sha256(fp_im) + + # convert to thread pool + sha256s = [] # ? + pool = ThreadPool(opt_threads) + with tqdm(total=len(fp_ims)) as pbar: + sha256s = pool.map(as_sha256, fp_ims) + pbar.close() + + + # ---------------------------------------------------------------- + # convert data to dict + + data = [] + for sha256, fp_im in zip(sha256s, fp_ims): + fpp_im = Path(fp_im) + subdir = str(fpp_im.parent.relative_to(fp_in)) + + if opt_identity: + subdirs = subdir.split('/') + if not len(subdirs) > 0: + log.error(f'Could not split subdir: "{subdir}. Try different option for "--identity"') + log.error('exiting') + return + if opt_identity == 'subdir': + identity = subdirs[0] # use first/only part + elif opt_identity == 'subdir_head': + identity = subdirs[0] # use first part of subdir path + elif opt_identity == 'subdir_tail': + identity = subdirs[-1] # use last part of subdir path + else: + identity = '' + + data.append({ + 'subdir': subdir, + 'fn': fpp_im.stem, + 'ext': fpp_im.suffix.replace('.',''), + 'sha256': sha256, + 'uuid': uuid.uuid4(), + 'identity_key': identity + }) + + log.info(f'adding identity index using: "{opt_identity}". This may take a while...') + # convert dict to DataFrame + df_records = pd.DataFrame.from_dict(data) + # sort based on identity_key + df_records = df_records.sort_values(by=['identity_key'], ascending=True) + # add new column for identity + df_records['identity_index'] = [-1] * len(df_records) + # populate the identity_index + df_records_identity_groups = df_records.groupby('identity_key') + # enumerate groups to create identity indices + for identity_index, df_records_identity_group_tuple in enumerate(df_records_identity_groups): + identity_key, df_records_identity_group = df_records_identity_group_tuple + for ds_record in df_records_identity_group.itertuples(): + df_records.at[ds_record.Index, 'identity_index'] = identity_index + # reset index after being sorted + df_records = df_records.reset_index(drop=True) + df_records.index.name = 'index' # reassign 'index' as primary key column + # write to CSV + file_utils.mkdirs(fp_out) + df_records.to_csv(fp_out) + # done + log.info(f'wrote rows: {len(df_records)} to {fp_out}') \ No newline at end of file diff --git a/megapixels/commands/datasets/s3.py b/megapixels/commands/datasets/s3.py deleted file mode 100644 index 7769896b..00000000 --- a/megapixels/commands/datasets/s3.py +++ /dev/null @@ -1,47 +0,0 @@ -import click - -from app.settings import types -from app.utils import click_utils -from app.settings import app_cfg as cfg - -s3_dirs = {'media': cfg.S3_MEDIA_ROOT, 'metadata': cfg.S3_METADATA_ROOT} - -@click.command() -@click.option('-i', '--input', 'opt_fps_in', required=True, multiple=True, - help='Input directory') -@click.option('--name', 'opt_dataset_name', required=True, - help='Dataset key (eg "lfw"') -@click.option('-a', '--action', 'opt_action', type=click.Choice(['sync', 'put']), default='sync', - help='S3 action') -@click.option('-t', '--type', 'opt_type', type=click.Choice(s3_dirs.keys()), required=True, - help='S3 location') -@click.option('--dry-run', 'opt_dryrun', is_flag=True, default=False) -@click.pass_context -def cli(ctx, opt_fps_in, opt_dataset_name, opt_action, opt_type, opt_dryrun): - """Syncs files with S3/spaces server""" - - from os.path import join - from pathlib import Path - - from tqdm import tqdm - import pandas as pd - import subprocess - - from app.utils import logger_utils, file_utils - - # ------------------------------------------------- - # init here - - log = logger_utils.Logger.getLogger() - for opt_fp_in in opt_fps_in: - dir_dst = join(s3_dirs[opt_type], opt_dataset_name, '') - if Path(opt_fp_in).is_dir(): - fp_src = join(opt_fp_in, '') # add trailing slashes - else: - fp_src = join(opt_fp_in) - cmd = ['s3cmd', opt_action, fp_src, dir_dst, '-P', '--follow-symlinks'] - log.info(' '.join(cmd)) - if not opt_dryrun: - subprocess.call(cmd) - - \ No newline at end of file diff --git a/megapixels/commands/datasets/s3_sync.py b/megapixels/commands/datasets/s3_sync.py new file mode 100644 index 00000000..3098d9be --- /dev/null +++ b/megapixels/commands/datasets/s3_sync.py @@ -0,0 +1,57 @@ +import click + +from app.settings import types +from app.utils import click_utils +from app.settings import app_cfg as cfg + +s3_dirs = {'media': cfg.S3_MEDIA_URL, 'metadata': cfg.S3_METADATA_URL} + +@click.command() +@click.option('--data_store', 'opt_data_store', + type=cfg.DataStoreVar, + default=click_utils.get_default(types.DataStore.SSD), + show_default=True, + help=click_utils.show_help(types.Dataset)) +@click.option('--dataset', 'opt_dataset', + type=cfg.DatasetVar, + required=True, + show_default=True, + help=click_utils.show_help(types.Dataset)) +@click.option('-t', '--type', 'opt_type', type=click.Choice(s3_dirs.keys()), required=True, + help='S3 location') +@click.option('--dry-run', 'opt_dryrun', is_flag=True, default=False) +@click.pass_context +def cli(ctx, opt_data_store, opt_dataset, opt_type, opt_dryrun): + """Syncs files with S3/spaces server""" + + from os.path import join + from pathlib import Path + + from tqdm import tqdm + import pandas as pd + import subprocess + + from app.utils import logger_utils, file_utils + from app.models.data_store import DataStore + + # ------------------------------------------------- + # init here + + log = logger_utils.Logger.getLogger() + + # set data_store + data_store = DataStore(opt_data_store, opt_dataset) + dataset_name = opt_dataset.name.lower() + if opt_type == 'media': + dir_src = join(data_store.uuid_dir(), '') + dir_dst = join(s3_dirs[opt_type], dataset_name, '') + elif opt_type == 'metadata': + dir_src = join(data_store.metadata_dir(), '') + dir_dst = join(s3_dirs[opt_type], dataset_name, '') + + cmd = ['s3cmd', 'sync', dir_src, dir_dst, '-P', '--follow-symlinks'] + log.info(' '.join(cmd)) + if not opt_dryrun: + subprocess.call(cmd) + + \ No newline at end of file diff --git a/megapixels/commands/datasets/symlink.py b/megapixels/commands/datasets/symlink.py deleted file mode 100644 index 70ec6c46..00000000 --- a/megapixels/commands/datasets/symlink.py +++ /dev/null @@ -1,45 +0,0 @@ -import click - -from app.settings import types -from app.utils import click_utils -from app.settings import app_cfg as cfg - -@click.command() -@click.option('-i', '--input', 'opt_fp_in', required=True, - help='Input records CSV') -@click.option('-m', '--media', 'opt_fp_media', required=True, - help='Input media directory') -@click.option('-o', '--output', 'opt_fp_out', required=True, - help='Output directory') -@click.pass_context -def cli(ctx, opt_fp_in, opt_fp_media, opt_fp_out): - """Symlinks images to new directory for S3""" - - import sys - import os - from os.path import join - from pathlib import Path - - from tqdm import tqdm - import pandas as pd - - from app.utils import logger_utils, file_utils - - # ------------------------------------------------- - # init here - - log = logger_utils.Logger.getLogger() - - df_records = pd.read_csv(opt_fp_in) - nrows = len(df_records) - - file_utils.mkdirs(opt_fp_out) - - for record_id, row in tqdm(df_records.iterrows(), total=nrows): - # make image path - df = df_records.iloc[record_id] - fpp_src = Path(join(opt_fp_media, df['subdir'], '{}.{}'.format(df['fn'], df['ext']))) - fpp_dst = Path(join(opt_fp_out, '{}.{}'.format(df['uuid'], df['ext']))) - fpp_dst.symlink_to(fpp_src) - - log.info('symlinked {:,} files'.format(nrows)) \ No newline at end of file diff --git a/megapixels/commands/datasets/symlink_uuid.py b/megapixels/commands/datasets/symlink_uuid.py new file mode 100644 index 00000000..7c5faa95 --- /dev/null +++ b/megapixels/commands/datasets/symlink_uuid.py @@ -0,0 +1,57 @@ +import click + +from app.settings import types +from app.utils import click_utils +from app.settings import app_cfg as cfg + +@click.command() +@click.option('-i', '--input', 'opt_fp_in', default=None, + help='Override enum input filename CSV') +@click.option('-o', '--output', 'opt_fp_out', default=None, + help='Override enum output filename CSV') +@click.option('--data_store', 'opt_data_store', + type=cfg.DataStoreVar, + default=click_utils.get_default(types.DataStore.SSD), + show_default=True, + help=click_utils.show_help(types.Dataset)) +@click.option('--dataset', 'opt_dataset', + type=cfg.DatasetVar, + required=True, + show_default=True, + help=click_utils.show_help(types.Dataset)) +@click.pass_context +def cli(ctx, opt_fp_in, opt_fp_out, opt_data_store, opt_dataset): + """Symlinks images to new directory for S3""" + + import sys + import os + from os.path import join + from pathlib import Path + + from tqdm import tqdm + import pandas as pd + + from app.utils import logger_utils, file_utils + from app.models.data_store import DataStore + + # ------------------------------------------------- + # init here + + log = logger_utils.Logger.getLogger() + + # set data_store + data_store = DataStore(opt_data_store, opt_dataset) + fp_records = data_store.metadata(types.Metadata.FILE_RECORD) + df_records = pd.read_csv(fp_records).set_index('index') + nrows = len(df_records) + + dir_out = data_store.uuid_dir() if opt_fp_out is None else opt_fp_out + file_utils.mkdirs(dir_out) + + for ds_record in tqdm(df_records.itertuples(), total=nrows): + # make image path + fp_src = data_store.face(ds_record.subdir, ds_record.fn, ds_record.ext) + fp_dst = data_store.face_uuid(ds_record.uuid, ds_record.ext) + Path(fp_dst).symlink_to(Path(fp_src)) + + log.info('symlinked {:,} files'.format(nrows)) \ No newline at end of file diff --git a/megapixels/commands/demo/face_search.py b/megapixels/commands/demo/face_search.py index 08b2323d..0452cc9d 100644 --- a/megapixels/commands/demo/face_search.py +++ b/megapixels/commands/demo/face_search.py @@ -45,10 +45,9 @@ def cli(ctx, opt_fp_in, opt_data_store, opt_dataset, opt_gpu): log = Logger.getLogger() # init face detection + detector = face_detector.DetectorDLIBHOG() # init face recognition - detector = face_detector.DetectorDLIBHOG() - # face recognition/vector recognition = face_recognition.RecognitionDLIB(gpu=opt_gpu) # load query image diff --git a/megapixels/notebooks/_local_scratch.ipynb b/megapixels/notebooks/_local_scratch.ipynb index 167b6ddd..cee17cba 100644 --- a/megapixels/notebooks/_local_scratch.ipynb +++ b/megapixels/notebooks/_local_scratch.ipynb @@ -1,161 +1,173 @@ { "cells": [ { - "cell_type": "code", - "execution_count": 1, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "import pandas as pd\n", - "import cv2 as cv\n", - "import numpy as np\n", - "%matplotlib inline\n", - "import matplotlib.pyplot as plt" + "# Scratch pad" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ - "import sys\n", "from glob import glob\n", "from os.path import join\n", "from pathlib import Path\n", + "import random\n", + "\n", + "import pandas as pd\n", + "import cv2 as cv\n", + "import numpy as np\n", + "%matplotlib inline\n", + "import matplotlib.pyplot as plt\n", + "\n", + "import sys\n", "sys.path.append('/work/megapixels_dev/megapixels')\n", "from app.models.bbox import BBox\n", - "#from app.utils import im_utils\n", - "import random" + "from app.utils import im_utils, file_utils" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ - "dir_ims = '/data_store_ssd/apps/megapixels/datasets/umd_faces/faces/'" + "a= [1]" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 6, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "0\n" - ] + "data": { + "text/plain": [ + "1" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "fp_ims = glob(join(dir_ims, '*.png'))\n", - "print(len(fp_ims))" + "a[-1]" ] }, { "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Help on function choice in module random:\n", - "\n", - "choice(self, seq)\n", - " Choose a random element from a non-empty sequence.\n", - "\n" - ] - } - ], - "source": [ - "help(random.sample)" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[1, 8, 0, 6, 3] True\n" - ] - } - ], - "source": [ - "a = list(range(0,10))\n", - "b = random.sample(a, 5)\n", - "print(b, len(set(b))==5)" - ] - }, - { - "cell_type": "code", - "execution_count": null, + "execution_count": 32, "metadata": {}, "outputs": [], "source": [ - "from random import randint\n", - "imu" + "fp_filepath = '/data_store_ssd/datasets/people/lfw/metadata/filepath.csv'\n", + "df_filepath = pd.read_csv(fp_filepath)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 48, "metadata": {}, "outputs": [], "source": [ - "import face_alignment\n", - "from skimage import io\n", - "\n", - "fa = face_alignment.FaceAlignment(face_alignment.LandmarksType._3D, flip_input=False, device='cuda')" + "image_index = 12467" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 55, "metadata": {}, - "outputs": [], - "source": [ - "fp_im = np.random.choice(fp_ims)\n", - "im = io.imread(fp_im)\n", - "preds = fa.get_landmarks(im)\n", - "print(preds[0])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "12474\n" + ] + }, + { + "data": { + "text/plain": [ + "index 12851\n", + "ext jpg\n", + "fn Vladimir_Putin_0029\n", + "subdir Vladimir_Putin\n", + "Name: 12474, dtype: object" + ] + }, + "execution_count": 55, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "import json" + "image_index += 1\n", + "print(image_index)\n", + "df_filepath.iloc[image_index]" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 56, "metadata": {}, "outputs": [], "source": [ - "print(len(preds[0]))\n" + "import imutils" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 57, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Help on function build_montages in module imutils.convenience:\n", + "\n", + "build_montages(image_list, image_shape, montage_shape)\n", + " ---------------------------------------------------------------------------------------------\n", + " author: Kyle Hounslow\n", + " ---------------------------------------------------------------------------------------------\n", + " Converts a list of single images into a list of 'montage' images of specified rows and columns.\n", + " A new montage image is started once rows and columns of montage image is filled.\n", + " Empty space of incomplete montage images are filled with black pixels\n", + " ---------------------------------------------------------------------------------------------\n", + " :param image_list: python list of input images\n", + " :param image_shape: tuple, size each image will be resized to for display (width, height)\n", + " :param montage_shape: tuple, shape of image montage (width, height)\n", + " :return: list of montage images in numpy array format\n", + " ---------------------------------------------------------------------------------------------\n", + " \n", + " example usage:\n", + " \n", + " # load single image\n", + " img = cv2.imread('lena.jpg')\n", + " # duplicate image 25 times\n", + " num_imgs = 25\n", + " img_list = []\n", + " for i in xrange(num_imgs):\n", + " img_list.append(img)\n", + " # convert image list into a montage of 256x256 images tiled in a 5x5 montage\n", + " montages = make_montages_of_images(img_list, (256, 256), (5, 5))\n", + " # iterate through montages and display\n", + " for montage in montages:\n", + " cv2.imshow('montage image', montage)\n", + " cv2.waitKey(0)\n", + " \n", + " ----------------------------------------------------------------------------------------------\n", + "\n" + ] + } + ], "source": [ - "with open('test.json', 'w') as fp:\n", - " json.dump(preds[0].tolist(), fp)" + "help(imutils.build_montages)" ] }, { @@ -182,7 +194,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.6.6" } }, "nbformat": 4, diff --git a/megapixels/notebooks/datasets/lfw/lfw_make_identity_csv.ipynb b/megapixels/notebooks/datasets/lfw/lfw_make_identity_csv.ipynb new file mode 100644 index 00000000..039614f0 --- /dev/null +++ b/megapixels/notebooks/datasets/lfw/lfw_make_identity_csv.ipynb @@ -0,0 +1,510 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Add identity ID to index" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from os.path import join\n", + "from pathlib import Path\n", + "import difflib\n", + "\n", + "from tqdm import tqdm_notebook as tqdm\n", + "import pandas as pd\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "# names\n", + "DATA_STORE = '/data_store_ssd/'\n", + "dir_dataset = 'datasets/people/lfw/metadata'" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "# split records into index and uuids\n", + "fp_identity_in = join(DATA_STORE, dir_dataset, 'identities_old.csv')\n", + "fp_identity_out = join(DATA_STORE, dir_dataset, 'identity_lookup.csv')\n", + "\n", + "df_identity = pd.read_csv(fp_identity_in).set_index('index')" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
namename_origdescriptiongenderimagesimage_index
index
0A. J. CookAJ CookCanadian actressf10
1AJ LamasAJ LamasAmerican actorm11
2Aaron EckhartAaron EckhartAmerican actorm12
3Aaron GuielAaron GuielProfessional baseball playerm13
4Aaron PattersonAaron PattersonAuthorm14
\n", + "
" + ], + "text/plain": [ + " name name_orig description gender \\\n", + "index \n", + "0 A. J. Cook AJ Cook Canadian actress f \n", + "1 AJ Lamas AJ Lamas American actor m \n", + "2 Aaron Eckhart Aaron Eckhart American actor m \n", + "3 Aaron Guiel Aaron Guiel Professional baseball player m \n", + "4 Aaron Patterson Aaron Patterson Author m \n", + "\n", + " images image_index \n", + "index \n", + "0 1 0 \n", + "1 1 1 \n", + "2 1 2 \n", + "3 1 3 \n", + "4 1 4 " + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_identity.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
namename_origdescriptiongenderimagesimage_indexsubdir
index
0A. J. CookAJ CookCanadian actressf10
1AJ LamasAJ LamasAmerican actorm11
\n", + "
" + ], + "text/plain": [ + " name name_orig description gender images image_index \\\n", + "index \n", + "0 A. J. Cook AJ Cook Canadian actress f 1 0 \n", + "1 AJ Lamas AJ Lamas American actor m 1 1 \n", + "\n", + " subdir \n", + "index \n", + "0 \n", + "1 " + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# associate each file with an identity\n", + "df_identity['subdir'] = [''] * len(df_identity)\n", + "df_identity.head(2)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ece5c11b90954b25b1f1e28fc2fe6b55", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(IntProgress(value=0, max=5749), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "for row in tqdm(df_identity.itertuples(), total=len(df_identity)):\n", + " name = row.name_orig\n", + " subdir = name.replace(' ','_')\n", + " df_identity.at[row.Index, 'subdir'] = subdir" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
namename_origdescriptiongenderimagesimage_indexsubdir
index
0A. J. CookAJ CookCanadian actressf10AJ_Cook
1AJ LamasAJ LamasAmerican actorm11AJ_Lamas
2Aaron EckhartAaron EckhartAmerican actorm12Aaron_Eckhart
3Aaron GuielAaron GuielProfessional baseball playerm13Aaron_Guiel
4Aaron PattersonAaron PattersonAuthorm14Aaron_Patterson
\n", + "
" + ], + "text/plain": [ + " name name_orig description gender \\\n", + "index \n", + "0 A. J. Cook AJ Cook Canadian actress f \n", + "1 AJ Lamas AJ Lamas American actor m \n", + "2 Aaron Eckhart Aaron Eckhart American actor m \n", + "3 Aaron Guiel Aaron Guiel Professional baseball player m \n", + "4 Aaron Patterson Aaron Patterson Author m \n", + "\n", + " images image_index subdir \n", + "index \n", + "0 1 0 AJ_Cook \n", + "1 1 1 AJ_Lamas \n", + "2 1 2 Aaron_Eckhart \n", + "3 1 3 Aaron_Guiel \n", + "4 1 4 Aaron_Patterson " + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_identity.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [], + "source": [ + "df_identity.to_csv(fp_identity_out)" + ] + }, + { + "cell_type": "code", + "execution_count": 138, + "metadata": {}, + "outputs": [], + "source": [ + "# make a clean index separate from files" + ] + }, + { + "cell_type": "code", + "execution_count": 145, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'AJ Lamas'" + ] + }, + "execution_count": 145, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#df_identies = pd.read_csv('identities.csv')\n", + "df_identities.iloc[1]['name']" + ] + }, + { + "cell_type": "code", + "execution_count": 149, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1 2 3 4\n" + ] + } + ], + "source": [ + "a = [1,2,3,4]\n", + "\n", + "print(*a)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:megapixels]", + "language": "python", + "name": "conda-env-megapixels-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/megapixels/notebooks/examples/face_recognition_demo.ipynb b/megapixels/notebooks/examples/face_recognition_demo.ipynb index 68c5f3b6..804c63b6 100644 --- a/megapixels/notebooks/examples/face_recognition_demo.ipynb +++ b/megapixels/notebooks/examples/face_recognition_demo.ipynb @@ -402,7 +402,9 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "import imutils" + ] }, { "cell_type": "code", -- cgit v1.2.3-70-g09d2