summaryrefslogtreecommitdiff
path: root/megapixels/commands
diff options
context:
space:
mode:
Diffstat (limited to 'megapixels/commands')
-rw-r--r--megapixels/commands/admin/rsync.py106
-rw-r--r--megapixels/commands/cv/crop.py104
-rw-r--r--megapixels/commands/cv/csv_to_faces.py105
-rw-r--r--megapixels/commands/cv/face_frames.py82
-rw-r--r--megapixels/commands/cv/faces_to_3dlm.py96
-rw-r--r--megapixels/commands/cv/faces_to_csv.py164
-rw-r--r--megapixels/commands/cv/mirror.py57
-rw-r--r--megapixels/commands/cv/resize.py128
-rw-r--r--megapixels/commands/cv/videos_to_frames.py73
-rw-r--r--megapixels/commands/datasets/50people.py129
-rw-r--r--megapixels/commands/datasets/megaface_flickr_api.py141
-rw-r--r--megapixels/commands/datasets/megaface_names.py65
-rw-r--r--megapixels/commands/datasets/sha256.py90
-rw-r--r--megapixels/commands/datasets/ytmu.py205
-rw-r--r--megapixels/commands/misc/compare_sres.py59
15 files changed, 1604 insertions, 0 deletions
diff --git a/megapixels/commands/admin/rsync.py b/megapixels/commands/admin/rsync.py
new file mode 100644
index 00000000..a821b460
--- /dev/null
+++ b/megapixels/commands/admin/rsync.py
@@ -0,0 +1,106 @@
+"""
+Parallel rsync media_records between drives
+For parallel rsync with media records, use vframe/commands/rsync
+"""
+
+import click
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+
+@click.command()
+@click.option('-i', '--input', 'dir_in', required=True,
+ help='Input directory')
+@click.option('-o', '--output', 'dir_out', required=True,
+ help='Output directory')
+@click.option('-t', '--threads', 'opt_threads', default=8,
+ help='Number of threads')
+@click.option('--validate/--no-validate', 'opt_validate', is_flag=True, default=False,
+ help='Validate files after copy')
+@click.option('--extract/--no-extract', 'opt_extract', is_flag=True, default=False,
+ help='Extract files after copy')
+@click.pass_context
+def cli(ctx, dir_in, dir_out, opt_threads, opt_validate, opt_extract):
+ """rsync folders"""
+
+ import os
+ from os.path import join
+ from pathlib import Path
+
+ # NB deactivate logger in imported module
+ import logging
+ logging.getLogger().addHandler(logging.NullHandler())
+ from parallel_sync import rsync
+
+ from app.settings.paths import Paths
+ from app.utils import logger_utils, file_utils
+
+ # -------------------------------------------------
+ # process here
+
+ log = logger_utils.Logger.getLogger()
+ log.info('RSYNC from {} to {}'.format(dir_in, dir_out))
+ log.info('opt_extract: {}'.format(opt_extract))
+ log.info('opt_validate: {}'.format(opt_validate))
+ log.info('opt_threads: {}'.format(opt_validate))
+
+ file_utils.mkdirs(dir_out)
+
+ rsync.copy(dir_in, dir_out, parallelism=opt_threads,
+ validate=opt_validate, extract=opt_extract)
+
+ log.info('done rsyncing')
+
+
+ # ---------------------------------------------------------------
+
+
+
+ # if dir_in:
+ # # use input filepath as source
+ # if not Path(dir_in).is_dir():
+ # log.error('{} is not a directory'.format(dir_in))
+ # ctx.exit()
+ # if not Path(dir_out).is_dir():
+ # ctx.log.error('{} is not a directory'.format(dir_out))
+ # return
+
+ # log.info('RSYNC from {} to {}'.format(dir_in, dir_out))
+ # log.debug('opt_validate: {}'.format(opt_validate))
+ # log.debug('opt_extract: {}'.format(opt_extract))
+ # # local_copy(paths, parallelism=10, extract=False, validate=False):
+ # file_utils.mkdirs(dir_out)
+ # rsync.copy(dir_in, dir_out, parallelism=opt_threads,
+ # validate=opt_validate, extract=opt_extract)
+ # else:
+ # log.debug('get paths')
+ # # use source mappings as rsync source
+ # if not opt_media_format:
+ # ctx.log.error('--media format not supplied for source mappings')
+ # return
+
+ # # ensure FILEPATH metadata exists
+ # # parallel-rsync accepts a list of tupes (src, dst)
+ # file_routes = []
+ # for chair_item in chair_items:
+ # item = chair_item.item
+ # sha256 = chair_item.item.sha256
+ # filepath_metadata = item.get_metadata(types.Metadata.FILEPATH)
+ # if not filepath_metadata:
+ # ctx.log.error('no FILEPATH metadata')
+ # return
+ # fp_media =
+ # src = join('')
+ # dir_media = Paths.media_dir(opt_media_format, data_store=opt_disk, verified=ctx.opts['verified'])
+ # dst = join('')
+ # file_routes.append((src, dst))
+
+ # ctx.log.debug('dir_media: {}'.format(dir_media))
+ # return
+
+ # # -------------------------------------------------
+
+ # # send back to sink
+ # for chair_item in chair_items:
+ # sink.send(chair_item)
diff --git a/megapixels/commands/cv/crop.py b/megapixels/commands/cv/crop.py
new file mode 100644
index 00000000..778be0c4
--- /dev/null
+++ b/megapixels/commands/cv/crop.py
@@ -0,0 +1,104 @@
+"""
+Crop images to prepare for training
+"""
+
+import click
+from PIL import Image, ImageOps, ImageFilter, ImageDraw
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+
+@click.command()
+@click.option('-i', '--input', 'opt_dir_in', required=True,
+ help='Input directory')
+@click.option('-o', '--output', 'opt_dir_out', required=True,
+ help='Output directory')
+@click.option('-e', '--ext', 'opt_ext',
+ default='jpg', type=click.Choice(['jpg', 'png']),
+ help='File glob ext')
+@click.option('--size', 'opt_size',
+ type=(int, int), default=(256, 256),
+ help='Output image size')
+@click.option('-t', '--crop-type', 'opt_crop_type',
+ default='center', type=click.Choice(['center', 'mirror', 'face', 'person', 'none']),
+ help='Force fit image center location')
+@click.pass_context
+def cli(ctx, opt_dir_in, opt_dir_out, opt_ext, opt_size, opt_crop_type):
+ """Crop, mirror images"""
+
+ import os
+ from os.path import join
+ from pathlib import Path
+ from glob import glob
+ from tqdm import tqdm
+
+
+ from app.utils import logger_utils, file_utils, im_utils
+
+ # -------------------------------------------------
+ # process here
+
+ log = logger_utils.Logger.getLogger()
+ log.info('crop images')
+
+ # get list of files to process
+ fp_ims = glob(join(opt_dir_in, '*.{}'.format(opt_ext)))
+ log.debug('files: {}'.format(len(fp_ims)))
+
+ # ensure output dir exists
+ file_utils.mkdirs(opt_dir_out)
+
+ for fp_im in tqdm(fp_ims):
+ im = process_crop(fp_im, opt_size, opt_crop_type)
+ fp_out = join(opt_dir_out, Path(fp_im).name)
+ im.save(fp_out)
+
+
+def process_crop(fp_im, opt_size, crop_type):
+ im = Image.open(fp_im)
+ if crop_type == 'center':
+ im = crop_square_fit(im, opt_size)
+ elif crop_type == 'mirror':
+ im = mirror_crop_square(im, opt_size)
+ return im
+
+def crop_square_fit(im, size, center=(0.5, 0.5)):
+ return ImageOps.fit(im, size, method=Image.BICUBIC, centering=center)
+
+def mirror_crop_square(im, size):
+ # force to even dims
+ if im.size[0] % 2 or im.size[1] % 2:
+ im = ImageOps.fit(im, ((im.size[0] // 2) * 2, (im.size[1] // 2) * 2))
+
+ # create new square image
+ min_size, max_size = (min(im.size), max(im.size))
+ orig_w, orig_h = im.size
+ margin = (max_size - min_size) // 2
+ w, h = (max_size, max_size)
+ im_new = Image.new('RGB', (w, h), color=(0, 0, 0))
+
+ #crop (l, t, r, b)
+ if orig_w > orig_h:
+ # landscape, mirror expand T/B
+ im_top = ImageOps.mirror(im.crop((0, 0, margin, w)))
+ im_bot = ImageOps.mirror(im.crop((orig_h - margin, 0, orig_h, w)))
+ im_new.paste(im_top, (0, 0))
+ im_new.paste(im, (margin, 0, orig_h + margin, w))
+ im_new.paste(im_bot, (h - margin, 0))
+ elif orig_h > orig_w:
+ # portrait, mirror expand L/R
+ im_left = ImageOps.mirror(im.crop((0, 0, margin, h)))
+ im_right = ImageOps.mirror(im.crop((orig_w - margin, 0, orig_w, h)))
+ im_new.paste(im_left, (0, 0))
+ im_new.paste(im, (margin, 0, orig_w + margin, h))
+ im_new.paste(im_right, (w - margin, 0))
+
+ return im_new.resize(size)
+
+
+def center_crop_face():
+ pass
+
+def center_crop_person():
+ pass \ No newline at end of file
diff --git a/megapixels/commands/cv/csv_to_faces.py b/megapixels/commands/cv/csv_to_faces.py
new file mode 100644
index 00000000..64c8b965
--- /dev/null
+++ b/megapixels/commands/cv/csv_to_faces.py
@@ -0,0 +1,105 @@
+"""
+Reads in CSV of ROIs and extracts facial regions with padding
+"""
+
+import click
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+
+@click.command()
+@click.option('-i', '--input', 'opt_fp_in', required=True,
+ help='Input CSV')
+@click.option('-m', '--media', 'opt_dir_media', required=True,
+ help='Input image/video directory')
+@click.option('-o', '--output', 'opt_dir_out', required=True,
+ help='Output directory for extracted ROI images')
+@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None),
+ help='Slice list of files')
+@click.option('--padding', 'opt_padding', default=0.25,
+ help='Facial padding as percentage of face width')
+@click.option('--ext', 'opt_ext_out', default='png', type=click.Choice(['jpg', 'png']),
+ help='Output image type')
+@click.option('--min', 'opt_min', default=(60, 60),
+ help='Minimum original face size')
+@click.pass_context
+def cli(ctx, opt_fp_in, opt_dir_media, opt_dir_out, opt_slice,
+ opt_padding, opt_ext_out, opt_min):
+ """Converts ROIs to images"""
+
+ import os
+ from os.path import join
+ from pathlib import Path
+ from glob import glob
+
+ from tqdm import tqdm
+ import numpy as np
+ from PIL import Image, ImageOps, ImageFilter, ImageDraw
+ import cv2 as cv
+ import pandas as pd
+
+ from app.utils import logger_utils, file_utils, im_utils
+ from app.models.bbox import BBox
+
+ # -------------------------------------------------
+ # process here
+ log = logger_utils.Logger.getLogger()
+
+ df_rois = pd.read_csv(opt_fp_in, dtype={'subdir': str, 'fn': str})
+ if opt_slice:
+ df_rois = df_rois[opt_slice[0]:opt_slice[1]]
+
+ log.info('Processing {:,} rows'.format(len(df_rois)))
+
+ file_utils.mkdirs(opt_dir_out)
+
+ df_rois_grouped = df_rois.groupby(['fn']) # group by fn/filename
+ groups = df_rois_grouped.groups
+ skipped = []
+
+ for group in tqdm(groups):
+ # get image
+ group_rows = df_rois_grouped.get_group(group)
+
+ row = group_rows.iloc[0]
+ fp_im = join(opt_dir_media, str(row['subdir']), '{fn}.{ext}'.format(**row)) # TODO change to ext
+ try:
+ im = Image.open(fp_im).convert('RGB')
+ im.verify()
+ except Exception as e:
+ log.warn('Could not open: {}'.format(fp_im))
+ log.error(e)
+ continue
+
+ for idx, roi in group_rows.iterrows():
+ # get bbox to im dimensions
+ xywh = [roi['x'], roi['y'], roi['w'] , roi['h']]
+ bbox = BBox.from_xywh(*xywh)
+ dim = im.size
+ bbox_dim = bbox.to_dim(dim)
+ # expand
+ opt_padding_px = int(opt_padding * bbox_dim.width)
+ bbox_dim_exp = bbox_dim.expand_dim(opt_padding_px, dim)
+ # crop
+ x1y2 = bbox_dim_exp.pt_tl + bbox_dim_exp.pt_br
+ im_crop = im.crop(box=x1y2)
+
+ # strip exif, create new image and paste data
+ im_crop_data = list(im_crop.getdata())
+ im_crop_no_exif = Image.new(im_crop.mode, im_crop.size)
+ im_crop_no_exif.putdata(im_crop_data)
+
+ # save
+ idx_zpad = file_utils.zpad(idx, zeros=3)
+ subdir = '' if roi['subdir'] == '.' else '{}_'.format(roi['subdir'])
+ subdir = subdir.replace('/', '_')
+ fp_im_out = join(opt_dir_out, '{}{}{}.{}'.format(subdir, roi['fn'], idx_zpad, opt_ext_out))
+ # threshold size and save
+ if im_crop_no_exif.size[0] < opt_min[0] or im_crop_no_exif.size[1] < opt_min[1]:
+ skipped.append(fp_im_out)
+ log.info('Face too small: {}, idx: {}'.format(fp_im, idx))
+ else:
+ im_crop_no_exif.save(fp_im_out)
+
+ log.info('Skipped {:,} images'.format(len(skipped)))
diff --git a/megapixels/commands/cv/face_frames.py b/megapixels/commands/cv/face_frames.py
new file mode 100644
index 00000000..76f23af1
--- /dev/null
+++ b/megapixels/commands/cv/face_frames.py
@@ -0,0 +1,82 @@
+from glob import glob
+import os
+from os.path import join
+from pathlib import Path
+
+import click
+
+
+
+
+@click.command()
+@click.option('-i', '--input', 'opt_fp_in', required=True,
+ help='Input directory to glob')
+@click.option('-o', '--output', 'opt_fp_out', required=True,
+ help='Output directory for face frames')
+@click.option('--size', 'opt_size',
+ type=(int, int), default=(300, 300),
+ help='Output image size')
+@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None),
+ help='Slice list of files')
+@click.pass_context
+def cli(ctx, opt_fp_in, opt_fp_out, opt_size, opt_slice):
+ """Split video to face frames"""
+
+ from tqdm import tqdm
+ import dlib
+ import pandas as pd
+ from PIL import Image, ImageOps, ImageFilter
+ import cv2 as cv
+ import numpy as np
+
+ from app.processors import face_detector
+ from app.utils import logger_utils, file_utils, im_utils
+ from app.settings import types
+ from app.utils import click_utils
+ from app.settings import app_cfg as cfg
+ from app.models.bbox import BBox
+
+ log = logger_utils.Logger.getLogger()
+
+ # -------------------------------------------------
+ # process
+
+ detector = face_detector.DetectorDLIBCNN()
+
+ # get file list
+ fp_videos = glob(join(opt_fp_in, '*.mp4'))
+ fp_videos += glob(join(opt_fp_in, '*.webm'))
+ fp_videos += glob(join(opt_fp_in, '*.mkv'))
+
+ min_distance_per = .025 # minimum distance percentage to save new face image
+ face_interval = 5
+ frame_interval_count = 0
+ frame_count = 0
+ bbox_prev = BBox(0,0,0,0)
+ file_utils.mkdirs(opt_fp_out)
+ dnn_size = opt_size
+ max_dim = max(dnn_size)
+ px_thresh = int(max_dim * min_distance_per)
+
+ for fp_video in tqdm(fp_videos):
+ # load video
+ video = cv.VideoCapture(fp_video)
+ # iterate through frames
+ while video.isOpened():
+ res, frame = video.read()
+ if not res:
+ break
+ # increment frames, save frame if interval has passed
+ frame_count += 1 # for naming
+ frame_interval_count += 1 # for interval
+ bboxes = detector.detect(frame, opt_size=dnn_size, opt_pyramids=0)
+ if len(bboxes) > 0 and frame_interval_count >= face_interval:
+ dim = frame.shape[:2][::-1]
+ d = bboxes[0].to_dim(dim).distance(bbox_prev)
+ if d > px_thresh:
+ # save frame
+ zfc = file_utils.zpad(frame_count)
+ fp_frame = join(opt_fp_out, '{}_{}.jpg'.format(Path(fp_video).stem, zfc))
+ cv.imwrite(fp_frame, frame)
+ frame_interval_count = 0
+ bbox_prev = bboxes[0]
diff --git a/megapixels/commands/cv/faces_to_3dlm.py b/megapixels/commands/cv/faces_to_3dlm.py
new file mode 100644
index 00000000..658d4484
--- /dev/null
+++ b/megapixels/commands/cv/faces_to_3dlm.py
@@ -0,0 +1,96 @@
+"""
+Crop images to prepare for training
+"""
+
+import click
+# from PIL import Image, ImageOps, ImageFilter, ImageDraw
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+
+color_filters = {'color': 1, 'gray': 2, 'all': 3}
+
+@click.command()
+@click.option('-i', '--input', 'opt_dirs_in', required=True, multiple=True,
+ help='Input directory')
+@click.option('-o', '--output', 'opt_fp_out', required=True,
+ help='Output CSV')
+@click.option('-e', '--ext', 'opt_ext',
+ default='jpg', type=click.Choice(['jpg', 'png']),
+ help='File glob ext')
+@click.option('--size', 'opt_size',
+ type=(int, int), default=(300, 300),
+ help='Output image size')
+@click.option('-g', '--gpu', 'opt_gpu', default=0,
+ help='GPU index')
+@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None),
+ help='Slice list of files')
+@click.option('--recursive/--no-recursive', 'opt_recursive', is_flag=True, default=False,
+ help='Use glob recursion (slower)')
+@click.option('-f', '--force', 'opt_force', is_flag=True,
+ help='Force overwrite file')
+@click.pass_context
+def cli(ctx, opt_dirs_in, opt_fp_out, opt_ext, opt_size, opt_gpu, opt_slice,
+ opt_recursive, opt_force):
+ """Converts face imges to 3D landmarks"""
+
+ import sys
+ import os
+ from os.path import join
+ from pathlib import Path
+ from glob import glob
+
+ from tqdm import tqdm
+ import numpy as np
+ import dlib # must keep a local reference for dlib
+ import cv2 as cv
+ import pandas as pd
+ from face_alignment import FaceAlignment, LandmarksType
+ from skimage import io
+
+ from app.utils import logger_utils, file_utils
+ from app.processors import face_detector
+
+ # -------------------------------------------------
+ # init here
+
+ log = logger_utils.Logger.getLogger()
+
+ if not opt_force and Path(opt_fp_out).exists():
+ log.error('File exists. Use "-f / --force" to overwite')
+ return
+
+ device = 'cuda' if opt_gpu > -1 else 'cpu'
+ fa = FaceAlignment(LandmarksType._3D, flip_input=False, device=device)
+
+ # get list of files to process
+ fp_ims = []
+ for opt_dir_in in opt_dirs_in:
+ if opt_recursive:
+ fp_glob = join(opt_dir_in, '**/*.{}'.format(opt_ext))
+ fp_ims += glob(fp_glob, recursive=True)
+ else:
+ fp_glob = join(opt_dir_in, '*.{}'.format(opt_ext))
+ fp_ims += glob(fp_glob)
+ log.debug(fp_glob)
+
+
+ if opt_slice:
+ fp_ims = fp_ims[opt_slice[0]:opt_slice[1]]
+ log.debug('processing {:,} files'.format(len(fp_ims)))
+
+
+ data = {}
+
+ for fp_im in tqdm(fp_ims):
+ fpp_im = Path(fp_im)
+ im = io.imread(fp_im)
+ preds = fa.get_landmarks(im)
+ if preds and len(preds) > 0:
+ data[fpp_im.name] = preds[0].tolist()
+
+ # save date
+ file_utils.mkdirs(opt_fp_out)
+
+ file_utils.write_json(data, opt_fp_out, verbose=True) \ No newline at end of file
diff --git a/megapixels/commands/cv/faces_to_csv.py b/megapixels/commands/cv/faces_to_csv.py
new file mode 100644
index 00000000..07226c31
--- /dev/null
+++ b/megapixels/commands/cv/faces_to_csv.py
@@ -0,0 +1,164 @@
+"""
+Crop images to prepare for training
+"""
+
+import click
+# from PIL import Image, ImageOps, ImageFilter, ImageDraw
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+
+color_filters = {'color': 1, 'gray': 2, 'all': 3}
+
+@click.command()
+@click.option('-i', '--input', 'opt_dirs_in', required=True, multiple=True,
+ help='Input directory')
+@click.option('-o', '--output', 'opt_fp_out', required=True,
+ help='Output CSV')
+@click.option('-e', '--ext', 'opt_ext',
+ default='jpg', type=click.Choice(['jpg', 'png']),
+ help='File glob ext')
+@click.option('--size', 'opt_size',
+ type=(int, int), default=(300, 300),
+ help='Output image size')
+@click.option('-t', '--detector-type', 'opt_detector_type',
+ type=cfg.FaceDetectNetVar,
+ default=click_utils.get_default(types.FaceDetectNet.DLIB_CNN),
+ help=click_utils.show_help(types.FaceDetectNet))
+@click.option('-g', '--gpu', 'opt_gpu', default=0,
+ help='GPU index')
+@click.option('--conf', 'opt_conf_thresh', default=0.85, type=click.FloatRange(0,1),
+ help='Confidence minimum threshold')
+@click.option('--pyramids', 'opt_pyramids', default=0, type=click.IntRange(0,4),
+ help='Number pyramids to upscale for DLIB detectors')
+@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None),
+ help='Slice list of files')
+@click.option('--display/--no-display', 'opt_display', is_flag=True, default=False,
+ help='Display detections to debug')
+@click.option('--recursive/--no-recursive', 'opt_recursive', is_flag=True, default=False,
+ help='Use glob recursion (slower)')
+@click.option('-f', '--force', 'opt_force', is_flag=True,
+ help='Force overwrite file')
+@click.option('--color', 'opt_color_filter',
+ type=click.Choice(color_filters.keys()), default='color',
+ help='Filter to keep color or grayscale images (color = keep color')
+@click.pass_context
+def cli(ctx, opt_dirs_in, opt_fp_out, opt_ext, opt_size, opt_detector_type,
+ opt_gpu, opt_conf_thresh, opt_pyramids, opt_slice, opt_display, opt_recursive, opt_force, opt_color_filter):
+ """Converts frames with faces to CSV of ROIs"""
+
+ import sys
+ import os
+ from os.path import join
+ from pathlib import Path
+ from glob import glob
+
+ from tqdm import tqdm
+ import numpy as np
+ import dlib # must keep a local reference for dlib
+ import cv2 as cv
+ import pandas as pd
+
+ from app.utils import logger_utils, file_utils, im_utils
+ from app.processors import face_detector
+
+ # -------------------------------------------------
+ # init here
+
+ log = logger_utils.Logger.getLogger()
+
+ if not opt_force and Path(opt_fp_out).exists():
+ log.error('File exists. Use "-f / --force" to overwite')
+ return
+
+ if opt_detector_type == types.FaceDetectNet.CVDNN:
+ detector = face_detector.DetectorCVDNN()
+ elif opt_detector_type == types.FaceDetectNet.DLIB_CNN:
+ detector = face_detector.DetectorDLIBCNN(opt_gpu)
+ elif opt_detector_type == types.FaceDetectNet.DLIB_HOG:
+ detector = face_detector.DetectorDLIBHOG()
+ elif opt_detector_type == types.FaceDetectNet.HAAR:
+ log.error('{} not yet implemented'.format(opt_detector_type.name))
+ return
+
+
+ # -------------------------------------------------
+ # process here
+ color_filter = color_filters[opt_color_filter]
+
+ # get list of files to process
+ fp_ims = []
+ for opt_dir_in in opt_dirs_in:
+ if opt_recursive:
+ fp_glob = join(opt_dir_in, '**/*.{}'.format(opt_ext))
+ fp_ims += glob(fp_glob, recursive=True)
+ else:
+ fp_glob = join(opt_dir_in, '*.{}'.format(opt_ext))
+ fp_ims += glob(fp_glob)
+ log.debug(fp_glob)
+
+
+ if opt_slice:
+ fp_ims = fp_ims[opt_slice[0]:opt_slice[1]]
+ log.debug('processing {:,} files'.format(len(fp_ims)))
+
+
+ data = []
+
+ for fp_im in tqdm(fp_ims):
+ im = cv.imread(fp_im)
+
+ # filter out color or grayscale iamges
+ if color_filter != color_filters['all']:
+ try:
+ is_gray = im_utils.is_grayscale(im)
+ if is_gray and color_filter != color_filters['gray']:
+ log.debug('Skipping grayscale image: {}'.format(fp_im))
+ continue
+ except Exception as e:
+ log.error('Could not check grayscale: {}'.format(fp_im))
+ continue
+
+ try:
+ bboxes = detector.detect(im, opt_size=opt_size, opt_pyramids=opt_pyramids)
+ except Exception as e:
+ log.error('could not detect: {}'.format(fp_im))
+ log.error('{}'.format(e))
+ fpp_im = Path(fp_im)
+ subdir = str(fpp_im.parent.relative_to(opt_dir_in))
+
+ for bbox in bboxes:
+ roi = {
+ 'fn': fpp_im.stem,
+ 'ext': fpp_im.suffix.replace('.',''),
+ 'x': bbox.x,
+ 'y': bbox.y,
+ 'w': bbox.w,
+ 'h': bbox.h,
+ 'image_height': im.shape[0],
+ 'image_width': im.shape[1],
+ 'subdir': subdir}
+ bbox_dim = bbox.to_dim(im.shape[:2][::-1]) # w,h
+ data.append(roi)
+
+ # debug display
+ if opt_display and len(bboxes):
+ im_md = im_utils.resize(im, width=min(1200, opt_size[0]))
+ for bbox in bboxes:
+ bbox_dim = bbox.to_dim(im_md.shape[:2][::-1])
+ cv.rectangle(im_md, bbox_dim.pt_tl, bbox_dim.pt_br, (0,255,0), 3)
+ cv.imshow('', im_md)
+ while True:
+ k = cv.waitKey(1) & 0xFF
+ if k == 27 or k == ord('q'): # ESC
+ cv.destroyAllWindows()
+ sys.exit()
+ elif k != 255:
+ # any key to continue
+ break
+
+ # save date
+ file_utils.mkdirs(opt_fp_out)
+ df = pd.DataFrame.from_dict(data)
+ df.to_csv(opt_fp_out, index=False) \ No newline at end of file
diff --git a/megapixels/commands/cv/mirror.py b/megapixels/commands/cv/mirror.py
new file mode 100644
index 00000000..9ca1cac7
--- /dev/null
+++ b/megapixels/commands/cv/mirror.py
@@ -0,0 +1,57 @@
+"""
+Crop images to prepare for training
+"""
+
+import click
+import cv2 as cv
+from PIL import Image, ImageOps, ImageFilter
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+
+
+@click.command()
+@click.option('-i', '--input', 'opt_dir_in', required=True,
+ help='Input directory')
+@click.option('-o', '--output', 'opt_dir_out', required=True,
+ help='Output directory')
+@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None),
+ help='Slice the input list')
+@click.pass_context
+def cli(ctx, opt_dir_in, opt_dir_out, opt_slice):
+ """Mirror augment image directory"""
+
+ import os
+ from os.path import join
+ from pathlib import Path
+ from glob import glob
+ from tqdm import tqdm
+
+ from app.utils import logger_utils, file_utils, im_utils
+
+ # -------------------------------------------------
+ # init
+
+ log = logger_utils.Logger.getLogger()
+
+ # -------------------------------------------------
+ # process here
+
+ # get list of files to process
+ fp_ims = glob(join(opt_dir_in, '*.jpg'))
+ fp_ims += glob(join(opt_dir_in, '*.png'))
+
+ if opt_slice:
+ fp_ims = fp_ims[opt_slice[0]:opt_slice[1]]
+ log.info('processing {:,} files'.format(len(fp_ims)))
+
+ # ensure output dir exists
+ file_utils.mkdirs(opt_dir_out)
+
+ # resize and save images
+ for fp_im in tqdm(fp_ims):
+ im = Image.open(fp_im)
+ fpp_im = Path(fp_im)
+ fp_out = join(opt_dir_out, '{}_mirror{}'.format(fpp_im.stem, fpp_im.suffix))
+ im.save(fp_out) \ No newline at end of file
diff --git a/megapixels/commands/cv/resize.py b/megapixels/commands/cv/resize.py
new file mode 100644
index 00000000..f535c8b6
--- /dev/null
+++ b/megapixels/commands/cv/resize.py
@@ -0,0 +1,128 @@
+"""
+Crop images to prepare for training
+"""
+
+import click
+import cv2 as cv
+from PIL import Image, ImageOps, ImageFilter
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+
+"""
+Filter Q-Down Q-Up Speed
+NEAREST ⭐⭐⭐⭐⭐
+BOX ⭐ ⭐⭐⭐⭐
+BILINEAR ⭐ ⭐ ⭐⭐⭐
+HAMMING ⭐⭐ ⭐⭐⭐
+BICUBIC ⭐⭐⭐ ⭐⭐⭐ ⭐⭐
+LANCZOS ⭐⭐⭐⭐ ⭐⭐⭐⭐ ⭐
+"""
+methods = {
+ 'lanczos': Image.LANCZOS,
+ 'bicubic': Image.BICUBIC,
+ 'hamming': Image.HAMMING,
+ 'bileaner': Image.BILINEAR,
+ 'box': Image.BOX,
+ 'nearest': Image.NEAREST
+ }
+centerings = {
+ 'tl': (0.0, 0.0),
+ 'tc': (0.5, 0.0),
+ 'tr': (0.0, 0.0),
+ 'lc': (0.0, 0.5),
+ 'cc': (0.5, 0.5),
+ 'rc': (1.0, 0.5),
+ 'bl': (0.0, 1.0),
+ 'bc': (1.0, 0.5),
+ 'br': (1.0, 1.0)
+}
+
+@click.command()
+@click.option('-i', '--input', 'opt_dir_in', required=True,
+ help='Input directory')
+@click.option('-o', '--output', 'opt_dir_out', required=True,
+ help='Output directory')
+@click.option('-e', '--ext', 'opt_glob_ext',
+ default='png', type=click.Choice(['jpg', 'png']),
+ help='File glob ext')
+@click.option('--size', 'opt_size',
+ type=(int, int), default=(256, 256),
+ help='Output image size (square)')
+@click.option('--method', 'opt_scale_method',
+ type=click.Choice(methods.keys()),
+ default='lanczos',
+ help='Scaling method to use')
+@click.option('--equalize', 'opt_equalize', is_flag=True,
+ help='Equalize historgram')
+@click.option('--sharpen', 'opt_sharpen', is_flag=True,
+ help='Unsharp mask')
+@click.option('--center', 'opt_center', default='cc', type=click.Choice(centerings.keys()),
+ help='Crop focal point')
+@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None),
+ help='Slice the input list')
+@click.pass_context
+def cli(ctx, opt_dir_in, opt_dir_out, opt_glob_ext, opt_size, opt_scale_method,
+ opt_equalize, opt_sharpen, opt_center, opt_slice):
+ """Crop, mirror images"""
+
+ import os
+ from os.path import join
+ from pathlib import Path
+ from glob import glob
+ from tqdm import tqdm
+
+ from app.utils import logger_utils, file_utils, im_utils
+
+ # -------------------------------------------------
+ # init
+
+ log = logger_utils.Logger.getLogger()
+
+ centering = centerings[opt_center]
+
+ # -------------------------------------------------
+ # process here
+
+ # get list of files to process
+ fp_ims = glob(join(opt_dir_in, '*.{}'.format(opt_glob_ext)))
+ if opt_slice:
+ fp_ims = fp_ims[opt_slice[0]:opt_slice[1]]
+ log.info('processing {:,} files'.format(len(fp_ims)))
+
+ # set scale method
+ scale_method = methods[opt_scale_method]
+
+ # ensure output dir exists
+ file_utils.mkdirs(opt_dir_out)
+
+ # resize and save images
+ for fp_im in tqdm(fp_ims):
+ try:
+ im = Image.open(fp_im).convert('RGB')
+ im.verify()
+ except Exception as e:
+ log.warn('Could not open: {}'.format(fp_im))
+ log.error(e)
+ continue
+
+ im = ImageOps.fit(im, opt_size, method=scale_method, centering=centering)
+
+ if opt_equalize:
+ im_np = im_utils.pil2np(im)
+ im_np_eq = eq_hist_yuv(im_np)
+ im_np = cv.addWeighted(im_np_eq, 0.35, im_np, 0.65, 0)
+ im = im_utils.np2pil(im_np)
+
+ if opt_sharpen:
+ im = im.filter(ImageFilter.UnsharpMask)
+
+ fp_out = join(opt_dir_out, Path(fp_im).name)
+ im.save(fp_out)
+
+
+def eq_hist_yuv(im):
+ im_yuv = cv.cvtColor(im, cv.COLOR_BGR2YUV)
+ im_yuv[:,:,0] = cv.equalizeHist(im_yuv[:,:,0])
+ return cv.cvtColor(im_yuv, cv.COLOR_YUV2BGR)
diff --git a/megapixels/commands/cv/videos_to_frames.py b/megapixels/commands/cv/videos_to_frames.py
new file mode 100644
index 00000000..0b56c46a
--- /dev/null
+++ b/megapixels/commands/cv/videos_to_frames.py
@@ -0,0 +1,73 @@
+from glob import glob
+import os
+from os.path import join
+from pathlib import Path
+
+import click
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+from app.utils import logger_utils
+
+import dlib
+import pandas as pd
+from PIL import Image, ImageOps, ImageFilter
+from app.utils import file_utils, im_utils
+
+
+log = logger_utils.Logger.getLogger()
+
+@click.command()
+@click.option('-i', '--input', 'opt_fp_in', required=True,
+ help='Input directory')
+@click.option('-o', '--output', 'opt_fp_out', required=True,
+ help='Output directory')
+@click.option('--size', 'opt_size', default=(320, 240),
+ help='Inference size for face detection' )
+@click.option('--interval', 'opt_frame_interval', default=20,
+ help='Number of frames before saving next face')
+@click.pass_context
+def cli(ctx, opt_fp_in, opt_fp_out, opt_size, opt_frame_interval):
+ """Converts videos to frames with faces"""
+
+ # -------------------------------------------------
+ # process
+
+ from tqdm import tqdm
+ import cv2 as cv
+ from tqdm import tqdm
+ from app.processors import face_detector
+
+ detector = face_detector.DetectorDLIBCNN()
+
+ # get file list
+ fp_videos = glob(join(opt_fp_in, '*.mp4'))
+ fp_videos += glob(join(opt_fp_in, '*.webm'))
+ fp_videos += glob(join(opt_fp_in, '*.mkv'))
+
+ frame_interval_count = 0
+ frame_count = 0
+
+ file_utils.mkdirs(opt_fp_out)
+
+ for fp_video in tqdm(fp_videos):
+
+ video = cv.VideoCapture(fp_video)
+
+ while video.isOpened():
+ res, frame = video.read()
+ if not res:
+ break
+
+ frame_count += 1 # for naming
+ frame_interval_count += 1 # for interval
+
+ bboxes = detector.detect(frame, opt_size=opt_size, opt_pyramids=0)
+ if len(bboxes) > 0 and frame_interval_count >= opt_frame_interval:
+ # save frame
+ fname = file_utils.zpad(frame_count)
+ fp_frame = join(opt_fp_out, '{}_{}.jpg'.format(Path(fp_video).stem, fname))
+ cv.imwrite(fp_frame, frame)
+ frame_interval_count = 0
+
diff --git a/megapixels/commands/datasets/50people.py b/megapixels/commands/datasets/50people.py
new file mode 100644
index 00000000..fb35b2fe
--- /dev/null
+++ b/megapixels/commands/datasets/50people.py
@@ -0,0 +1,129 @@
+from glob import glob
+import os
+from os.path import join
+from pathlib import Path
+
+import click
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+from app.utils import logger_utils
+
+import dlib
+import pandas as pd
+from PIL import Image, ImageOps, ImageFilter
+from app.utils import file_utils, im_utils
+
+
+log = logger_utils.Logger.getLogger()
+
+@click.command()
+@click.option('-i', '--input', 'opt_fp_in', required=True,
+ help='Input directory')
+@click.option('-o', '--output', 'opt_fp_out',
+ help='Output directory')
+@click.option('--media', 'opt_dir_media',
+ help='Output directory')
+@click.option('--action', 'opt_action',
+ type=click.Choice(['download']),
+ default='info',
+ help='Command action')
+@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None),
+ help='Slice list of files')
+@click.pass_context
+def cli(ctx, opt_fp_in, opt_fp_out, opt_dir_media, opt_action, opt_slice):
+ """YTMU utils"""
+
+
+ from tqdm import tqdm
+
+ # -------------------------------------------------
+ # process
+
+ if opt_action == 'download':
+ # downloads video files with ytdl
+ handle_download(opt_fp_in, opt_fp_out, opt_slice)
+ elif opt_action == 'face_frames':
+ handle_face_frames(opt_fp_in, opt_fp_out, dir_media, opt_slice)
+
+
+
+
+
+def handle_face_frames(fp_in, dir_out, dir_videos):
+ if not dir_out or not dir_videos:
+ log.error('-o/--output and --videos required')
+ return
+
+ import cv2 as cv
+ from tqdm import tqdm
+ from app.processors import face_detector
+ detector = face_detector.DetectorDLIBCNN()
+
+ # get file list
+ fp_videos = glob(join(dir_videos, '*.mp4'))
+ fp_videos += glob(join(dir_videos, '*.webm'))
+ fp_videos += glob(join(dir_videos, '*.mkv'))
+
+ face_interval = 30
+ frame_interval_count = 0
+ frame_count = 0
+
+ file_utils.mkdirs(dir_out)
+
+ for fp_video in tqdm(fp_videos):
+ # log.debug('opening: {}'.format(fp_video))
+ video = cv.VideoCapture(fp_video)
+ while video.isOpened():
+ res, frame = video.read()
+ if not res:
+ break
+
+ frame_count += 1 # for naming
+ frame_interval_count += 1 # for interval
+ bboxes = detector.detect(frame, opt_size=(320, 240), opt_pyramids=0)
+ if len(bboxes) > 0 and frame_interval_count >= face_interval:
+ # save frame
+ fp_frame = join(dir_out, '{}_{}.jpg'.format(Path(fp_video).stem, file_utils.zpad(frame_count)))
+ cv.imwrite(fp_frame, frame)
+ frame_interval_count = 0
+
+
+def handle_download(fp_in, dir_out, opt_slice):
+ import youtube_dl
+ df = pd.read_csv(fp_in)
+ if opt_slice:
+ df = df[opt_slice[0]:opt_slice[1]]
+ df = df.fillna('')
+ fp_videos = glob(join(dir_out, '*.mp4'))
+ fp_videos += glob(join(dir_out, '*.webm'))
+ fp_videos += glob(join(dir_out, '*.mkv'))
+
+ ydl = youtube_dl.YoutubeDL({'outtmpl': join(dir_out, '') + '%(id)s.%(ext)s'})
+
+ for i, row in df.iterrows():
+ vid = str(row['youtube_id'])
+ if not vid:
+ vid = row['vimeo_id']
+ if vid:
+ vid = str(int(vid))
+ url = 'https://vimeo.com/{}'.format(vid)
+ else:
+ url = 'https://youtube.com/watch?v={}'.format(vid)
+ if not vid:
+ log.warn('no video id: {} for {}'.format(vid, row['city']))
+ continue
+
+ found = False
+ for fp_video in fp_videos:
+ if vid in fp_video:
+ #log.debug('skip: {}'.format(vid))
+ found = True
+
+ if not found:
+ try:
+ with ydl:
+ ydl.download([url])
+ except:
+ log.error('could not dl: {}'.format(vid))
diff --git a/megapixels/commands/datasets/megaface_flickr_api.py b/megapixels/commands/datasets/megaface_flickr_api.py
new file mode 100644
index 00000000..62232ab8
--- /dev/null
+++ b/megapixels/commands/datasets/megaface_flickr_api.py
@@ -0,0 +1,141 @@
+from glob import glob
+import os
+from os.path import join
+from pathlib import Path
+
+import click
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+from app.utils import logger_utils
+
+import dlib
+import pandas as pd
+from PIL import Image, ImageOps, ImageFilter
+from app.utils import file_utils, im_utils
+
+
+log = logger_utils.Logger.getLogger()
+
+@click.command()
+@click.option('-i', '--input', 'opt_fp_in', required=True,
+ help='Input directory')
+@click.option('-o', '--output', 'opt_fp_out',
+ help='Output directory')
+@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None),
+ help='Slice list of files')
+@click.option('-d', '--delay', 'opt_delay', default=None, type=int,
+ help='Delay between API calls to prevent rate-limiting')
+@click.option('--checkpoints', 'opt_checkpoints', is_flag=True,
+ help='Save checkpoints')
+@click.option('--api_key', 'opt_api_key', envvar='FLICKR_API_KEY')
+@click.option('--api_secret', 'opt_api_secret', envvar='FLICKR_API_SECRET')
+@click.option('--checkpoint_interval', 'opt_ckpt_interval', default=10000,
+ help='Save checkpoint interval')
+@click.pass_context
+def cli(ctx, opt_fp_in, opt_fp_out, opt_slice, opt_api_key, opt_api_secret,
+ opt_delay, opt_checkpoints, opt_ckpt_interval):
+ """Appends Flickr API info to CSV"""
+
+ from tqdm import tqdm
+ from glob import glob
+ import time
+ import flickr_api # pip install flickr_api
+ from flickr_api.flickrerrors import FlickrAPIError
+
+ # -------------------------------------------------
+ # process
+
+ if not opt_api_key or not opt_api_secret:
+ log.error('source .env vars for Flickr API and try again')
+ return
+
+ # init Flickr API
+ flickr_api.set_keys(api_key=opt_api_key, api_secret=opt_api_secret)
+
+ # reqd in CSV
+ df_ids = pd.read_csv(opt_fp_in)
+ if opt_slice:
+ df_ids = df_ids[opt_slice[0]:opt_slice[1]]
+
+ log.info('Processing: {:,} items'.format(len(df_ids)))
+
+ # iterate MegaFace IDs
+ identities = []
+
+ tqdm.pandas()
+
+ for idx, df_id in tqdm(df_ids.iterrows(), total=len(df_ids)):
+ # a = flickr_api.Person(id='123456789@N01')
+ df_id_dict = dict(df_id)
+
+ # append relevant data
+ try:
+ person = flickr_api.Person(id=df_id['nsid'])
+ info = person.getInfo()
+ df_id_dict.update( {
+ 'user_name': info.get('username', ''),
+ 'location': info.get('location', ''),
+ 'real_name': info.get('realname', ''),
+ 'time_zone': info.get('timezone', {}).get('timezone_id', ''),
+ 'time_first_photo': info.get('photos_info', {}).get('firstdatetaken'),
+ 'photos_count': info.get('photos_info', {}).get('count'),
+ 'description': info.get('description', ''),
+ 'id': info.get('id'),
+ 'path_alias': info.get('path_alias', ''),
+ 'is_pro': info.get('ispro', ''),
+ 'url_photos': info.get('photosurl', ''),
+ 'url_profile': info.get('photosurl', ''),
+ 'url_mobile': info.get('mobileurl', ''),
+ })
+ identities.append(df_id_dict)
+
+ except FlickrAPIError as e:
+ log.error(e)
+
+
+ if opt_checkpoints:
+ if (idx + 1) % opt_ckpt_interval == 0:
+ df = pd.DataFrame.from_dict(identities)
+ fpp_out = Path(opt_fp_out)
+ opt_fp_out_ckpt = join(fpp_out.parent, '{}_ckpt_{}.csv'.format(fpp_out.stem, file_utils.zpad(idx + 1)))
+ log.info('Saving checkpoint {:,} to {}'.format(idx + 1, opt_fp_out_ckpt))
+ df.to_csv(opt_fp_out_ckpt, index=False)
+
+ if opt_delay:
+ time.sleep(opt_delay)
+
+
+ df = pd.DataFrame.from_dict(identities)
+ df.to_csv(opt_fp_out, index=False)
+
+ log.info('Wrote: {:,} lines to {}'.format(len(df), opt_fp_out))
+
+
+"""
+Example API data:
+{'id': '7124086@N07',
+ 'nsid': '7124086@N07',
+ 'ispro': 1,
+ 'can_buy_pro': 0,
+ 'iconserver': '2325',
+ 'iconfarm': 3,
+ 'path_alias': 'shirleylin',
+ 'has_stats': '1',
+ 'pro_badge': 'standard',
+ 'expire': '0',
+ 'username': 'ShirleyLin',
+ 'realname': 'Shirley Lin',
+ 'location': 'Fremont, California, US',
+ 'timezone': {'label': 'Pacific Time (US & Canada); Tijuana',
+ 'offset': '-08:00',
+ 'timezone_id': 'PST8PDT'},
+ 'description': '',
+ 'photosurl': 'https://www.flickr.com/photos/shirleylin/',
+ 'profileurl': 'https://www.flickr.com/people/shirleylin/',
+ 'mobileurl': 'https://m.flickr.com/photostream.gne?id=7102756',
+ 'photos_info': {'firstdatetaken': '2004-05-24 12:12:15',
+ 'firstdate': '1172556588',
+ 'count': 9665}}
+""" \ No newline at end of file
diff --git a/megapixels/commands/datasets/megaface_names.py b/megapixels/commands/datasets/megaface_names.py
new file mode 100644
index 00000000..01e93e2d
--- /dev/null
+++ b/megapixels/commands/datasets/megaface_names.py
@@ -0,0 +1,65 @@
+from glob import glob
+import os
+from os.path import join
+from pathlib import Path
+
+import click
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+from app.utils import logger_utils
+
+import dlib
+import pandas as pd
+from PIL import Image, ImageOps, ImageFilter
+from app.utils import file_utils, im_utils
+
+
+log = logger_utils.Logger.getLogger()
+
+@click.command()
+@click.option('-i', '--input', 'opt_fp_in', required=True,
+ help='Input directory')
+@click.option('-o', '--output', 'opt_fp_out',
+ help='Output directory')
+@click.pass_context
+def cli(ctx, opt_fp_in, opt_fp_out):
+ """Creates CSV of NSIDs from MegaFace"""
+
+ from tqdm import tqdm
+ from glob import glob
+
+ # -------------------------------------------------
+ # process
+ fp_im_dirs = glob(join(opt_fp_in, '**/'), recursive=True)
+
+ log.info('Found {} directories'.format(len(fp_im_dirs)))
+
+ identities = {}
+
+ for fp_im_dir in tqdm(fp_im_dirs):
+ # 1234567@N05_identity_1
+ try:
+ dir_id_name = Path(fp_im_dir).name
+ nsid = dir_id_name.split('_')[0]
+ identity_num = dir_id_name.split('_')[2]
+ id_key = '{}_{}'.format(nsid, identity_num)
+ num_images = len(glob(join(fp_im_dir, '*.jpg')))
+ if not id_key in identities.keys():
+ identities[id_key] = {'nsid': nsid, 'identity': identity_num, 'images': num_images}
+ else:
+ identities[id_key]['images'] += num_images
+ except Exception as e:
+ continue
+
+ # convert to dict
+ identities_list = [v for k, v in identities.items()]
+ df = pd.DataFrame.from_dict(identities_list)
+
+ file_utils.mkdirs(opt_fp_out)
+
+ log.info('Wrote {} lines to {}'.format(len(df), opt_fp_out))
+ df.to_csv(opt_fp_out, index=False)
+
+
diff --git a/megapixels/commands/datasets/sha256.py b/megapixels/commands/datasets/sha256.py
new file mode 100644
index 00000000..c04fb504
--- /dev/null
+++ b/megapixels/commands/datasets/sha256.py
@@ -0,0 +1,90 @@
+import click
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+from app.utils.logger_utils import Logger
+
+log = Logger.getLogger()
+
+@click.command()
+@click.option('-i', '--input', 'opt_fp_in', required=True,
+ help='Input directory')
+@click.option('-o', '--output', 'opt_fp_out',
+ help='Output directory')
+@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None),
+ help='Slice list of files')
+@click.option('--recursive/--no-recursive', 'opt_recursive', is_flag=True, default=False,
+ help='Use glob recursion (slower)')
+@click.option('-t', '--threads', 'opt_threads', default=4,
+ help='Number of threads')
+@click.option('-f', '--force', 'opt_force', is_flag=True,
+ help='Force overwrite file')
+@click.pass_context
+def cli(ctx, opt_fp_in, opt_fp_out, opt_slice, opt_recursive, opt_threads, opt_force):
+ """Multithreading test"""
+
+ from glob import glob
+ from os.path import join
+ from pathlib import Path
+ import time
+ from multiprocessing.dummy import Pool as ThreadPool
+ import random
+
+ import pandas as pd
+ from tqdm import tqdm
+ from glob import glob
+
+ from app.utils import file_utils, im_utils
+
+
+ if not opt_force and Path(opt_fp_out).exists():
+ log.error('File exists. Use "-f / --force" to overwite')
+ return
+
+ fp_ims = []
+ for ext in ['jpg', 'png']:
+ if opt_recursive:
+ fp_glob = join(opt_fp_in, '**/*.{}'.format(ext))
+ fp_ims += glob(fp_glob, recursive=True)
+ else:
+ fp_glob = join(opt_fp_in, '*.{}'.format(ext))
+ fp_ims += glob(fp_glob)
+
+ if opt_slice:
+ fp_ims = fp_ims[opt_slice[0]:opt_slice[1]]
+
+ log.info('Processing {:,} images'.format(len(fp_ims)))
+
+ pbar = tqdm(total=100)
+
+ def as_sha256(fp_im):
+ pbar.update(1)
+ return file_utils.sha256(fp_im)
+
+ # multithread pool
+ st = time.time()
+ pool = ThreadPool(opt_threads)
+ with tqdm(total=len(fp_ims)) as pbar:
+ sha256s = pool.map(as_sha256, fp_ims)
+ pbar.close()
+
+ # convert data to dict
+ data = []
+ for i, fp_im in enumerate(fp_ims):
+ fpp_im = Path(fp_im)
+ subdir = str(fpp_im.parent.relative_to(opt_fp_in))
+ sha256 = sha256s[i]
+ data.append( {
+ 'sha256': sha256,
+ 'subdir': subdir,
+ 'fn': fpp_im.stem,
+ 'ext': fpp_im.suffix.replace('.','')
+ })
+
+ # save to CSV
+ df = pd.DataFrame.from_dict(data)
+ df.to_csv(opt_fp_out, index=False)
+
+ # timing
+ log.info('time: {:.2f}, theads: {}'.format(time.time() - st, opt_threads)) \ No newline at end of file
diff --git a/megapixels/commands/datasets/ytmu.py b/megapixels/commands/datasets/ytmu.py
new file mode 100644
index 00000000..66680ed0
--- /dev/null
+++ b/megapixels/commands/datasets/ytmu.py
@@ -0,0 +1,205 @@
+from glob import glob
+import os
+from os.path import join
+from pathlib import Path
+
+import click
+
+from app.settings import types
+from app.utils import click_utils
+from app.settings import app_cfg as cfg
+from app.utils import logger_utils
+
+import dlib
+import pandas as pd
+from PIL import Image, ImageOps, ImageFilter
+from app.utils import file_utils, im_utils
+
+
+log = logger_utils.Logger.getLogger()
+
+@click.command()
+@click.option('-i', '--input', 'opt_fp_in', required=True,
+ help='Input directory')
+@click.option('-o', '--output', 'opt_fp_out',
+ help='Output directory')
+@click.option('--videos', 'opt_dir_videos',
+ help='Output directory')
+@click.option('--action', 'opt_action',
+ type=click.Choice(['info', 'faces', 'rename', 'download', 'metadata', 'split_frames']),
+ default='info',
+ help='Command action')
+@click.pass_context
+def cli(ctx, opt_fp_in, opt_fp_out, opt_dir_videos, opt_action):
+ """YTMU utils"""
+
+
+ from tqdm import tqdm
+
+ # -------------------------------------------------
+ # process
+
+ if opt_action == 'metadata':
+ # downloads video metadata with ytdl
+ handle_metadata(opt_fp_in, opt_fp_out)
+ elif opt_action == 'download':
+ # downloads video files with ytdl
+ handle_download(opt_fp_in, opt_fp_out)
+ elif opt_action == 'info':
+ # converts original data file to clean CSV
+ handle_info()
+ elif opt_action == 'rename':
+ # rename the videos to video ID
+ handle_rename(opt_fp_in, opt_fp_out, opt_dir_videos)
+ elif opt_action == 'split_frames':
+ # rename the videos to video ID
+ handle_split_frames(opt_fp_in, opt_fp_out, opt_dir_videos)
+
+
+
+
+# ----------------------------------------------------
+# handlers
+
+def handle_split_frames(fp_in, dir_out, dir_videos):
+ if not dir_out or not dir_videos:
+ log.error('-o/--output and --videos required')
+ return
+ import cv2 as cv
+ from tqdm import tqdm
+ from app.processors import face_detector
+ detector = face_detector.DetectorDLIBCNN()
+
+ # get file list
+ fp_videos = glob(join(dir_videos, '*.mp4'))
+ fp_videos += glob(join(dir_videos, '*.webm'))
+ fp_videos += glob(join(dir_videos, '*.mkv'))
+ face_interval = 30
+ frame_interval_count = 0
+ frame_count = 0
+
+ file_utils.mkdirs(dir_out)
+
+ for fp_video in tqdm(fp_videos):
+ # log.debug('opening: {}'.format(fp_video))
+ video = cv.VideoCapture(fp_video)
+ while video.isOpened():
+ res, frame = video.read()
+ if not res:
+ break
+
+ frame_count += 1 # for naming
+ frame_interval_count += 1 # for interval
+ bboxes = detector.detect(frame, opt_size=(320, 240), opt_pyramids=0)
+ if len(bboxes) > 0 and frame_interval_count >= face_interval:
+ # save frame
+ fp_frame = join(dir_out, '{}_{}.jpg'.format(Path(fp_video).stem, file_utils.zpad(frame_count)))
+ cv.imwrite(fp_frame, frame)
+ frame_interval_count = 0
+
+
+def handle_metadata(fp_in, fp_out):
+
+ keys = ['description', 'average_rating', 'dislike_count', 'categories',
+ 'thumbnail', 'title', 'upload_date', 'uploader_url', 'uploader_id',
+ 'fps', 'height', 'width', 'like_count', 'license', 'tags']
+
+ import youtube_dl
+
+ ydl = youtube_dl.YoutubeDL({'outtmpl': '%(id)s%(ext)s'})
+
+ df = pd.read_csv(fp_in)
+ data_exp = []
+
+ for i, row in df.iterrows():
+ video_data = {'url': row['url'], 'id': row['id']}
+ try:
+ with ydl:
+ url = 'http://www.youtube.com/watch?v={}'.format(row['id'])
+ result = ydl.extract_info(url, download=False)
+ video = result['entries'][0] if 'entries' in result else result
+ for k in keys:
+ val = video[k]
+ if k == 'title':
+ log.debug(val)
+ if type(val) == list:
+ val = '; '.join(val)
+ if type(val) == str:
+ video_data[k] = str(val).replace(',',';')
+ # log.debug('video_data: {}'.format(video_data))
+ except Exception as e:
+ log.warn('video unavilable: {}'.format(row['url']))
+ log.error(e)
+ continue
+ data_exp.append(video_data)
+
+ df_exp = pd.DataFrame.from_dict(data_exp)
+ df_exp.to_csv(fp_out)
+
+
+def handle_download(fp_in, dir_out):
+ import youtube_dl
+ df = pd.read_csv(fp_in)
+ fp_videos = glob(join(dir_out, '*.mp4'))
+ fp_videos += glob(join(dir_out, '*.webm'))
+ fp_videos += glob(join(dir_out, '*.mkv'))
+
+ ydl = youtube_dl.YoutubeDL({'outtmpl': '%(id)s%(ext)s'})
+
+ for i, row in df.iterrows():
+ vid = row['id']
+ found = False
+ for fp_video in fp_videos:
+ if vid in fp_video:
+ log.debug('skip: {}'.format(vid))
+ found = True
+ if not found:
+ try:
+ with ydl:
+ ydl.download(['http://www.youtube.com/watch?v={}'.format(vid)])
+ except:
+ log.error('could not dl: {}'.format(vid))
+
+
+def handle_info(fp_in, fp_out):
+ if not fp_out:
+ log.error('--output required')
+ return
+ urls = file_utils.load_text(fp_in)
+ videos = []
+ for url in urls:
+ splits = url.split('v=')
+ try:
+ vid = splits[1]
+ vid = vid.split('&')[0]
+ videos.append({'url': url, 'id': vid})
+ except:
+ log.warn('no video id for {}'.format(url))
+ # convert to df
+ df = pd.DataFrame.from_dict(videos)
+ df.to_csv(opt_fp_out)
+
+
+def handle_rename(fp_in, fp_out, dir_videos):
+ import shutil
+
+ if not dir_videos:
+ log.error('--videos required')
+ return
+
+ fp_videos = glob(join(dir_videos, '*.mp4'))
+ fp_videos += glob(join(dir_videos, '*.webm'))
+ fp_videos += glob(join(dir_videos, '*.mkv'))
+
+ df = pd.read_csv(fp_in)
+
+ for i, row in df.iterrows():
+ vid = row['id']
+ fp_videos_copy = fp_videos.copy()
+ for fp_video in fp_videos:
+ if vid in fp_video:
+ dst = join(dir_videos, '{}{}'.format(vid, Path(fp_video).suffix))
+ shutil.move(fp_video, dst)
+ log.debug('move {} to {}'.format(fp_video, dst))
+ fp_videos.remove(fp_video)
+ break \ No newline at end of file
diff --git a/megapixels/commands/misc/compare_sres.py b/megapixels/commands/misc/compare_sres.py
new file mode 100644
index 00000000..b96570fe
--- /dev/null
+++ b/megapixels/commands/misc/compare_sres.py
@@ -0,0 +1,59 @@
+import click
+
+
+@click.command()
+@click.option('-i', '--orig', 'opt_dir_in_orig', required=True,
+ help='Input directory')
+@click.option('-n', '--new', 'opt_dir_in_new', required=True,
+ help='Input directory files to compare to')
+@click.pass_context
+def cli(ctx, opt_dir_in_orig, opt_dir_in_new):
+ """Compare quality of super resolution images"""
+
+ import os
+
+ import sys
+ from os.path import join
+ from pathlib import Path
+ from glob import glob
+
+ from random import randint
+ from PIL import Image, ImageOps, ImageFilter
+ from pathlib import Path
+ import cv2 as cv
+
+ from app.settings import types
+ from app.utils import click_utils
+ from app.settings import app_cfg as cfg
+ from app.utils import file_utils, im_utils, logger_utils
+
+ log = logger_utils.Logger.getLogger()
+
+ fp_ims = glob(join(opt_dir_in_new, '*.jpg'))
+ fp_ims += glob(join(opt_dir_in_new, '*.png'))
+
+ log.info('{}'.format(len(fp_ims)))
+
+ while True:
+ rn = randint(0, len(fp_ims) - 1)
+ fp_im_new = fp_ims[rn]
+ fp_im_orig = fp_im_new.replace(opt_dir_in_new, opt_dir_in_orig)
+ log.info('new: {}'.format(fp_im_new))
+ log.info('orig: {}'.format(fp_im_orig))
+
+ im_new = cv.imread(fp_im_new)
+ im_orig = cv.imread(fp_im_orig)
+
+ # show
+ cv.imshow('new', im_new)
+ cv.imshow('orig', im_orig)
+
+ # handle key io
+ k = cv.waitKey(0) & 0xFF
+ if k == 27 or k == ord('q'): # ESC
+ # exits the app
+ cv.destroyAllWindows()
+ sys.exit('Exiting because Q or ESC was pressed')
+ elif k == ord(' ') or k == 81 or k == 83:
+ continue
+
generated by cgit v1.2.3-70-g09d2 (git 2.51.0) at 2025-12-10 17:57:05 +0000