From 156790b383101756e2324dcde63415f00ba94a86 Mon Sep 17 00:00:00 2001 From: adamhrv Date: Sun, 4 Nov 2018 21:44:20 +0100 Subject: . --- README.md | 3 + environment.yml | 207 ++++++++ megapixels/admin/commands/rsync.py | 106 ++++ megapixels/app/models/__init__.py | 0 megapixels/app/models/bbox.py | 236 +++++++++ megapixels/app/models/click_factory.py | 145 ++++++ megapixels/app/processors/__init__.py | 0 megapixels/app/processors/face_detector.py | 103 ++++ megapixels/app/settings/__init__.py | 0 megapixels/app/settings/app_cfg.py | 90 ++++ megapixels/app/settings/types.py | 29 ++ megapixels/app/utils/__init__.py | 0 megapixels/app/utils/click_utils.py | 62 +++ megapixels/app/utils/file_utils.py | 400 +++++++++++++++ megapixels/app/utils/im_utils.py | 506 ++++++++++++++++++ megapixels/app/utils/logger_utils.py | 68 +++ megapixels/cli_admin.py | 36 ++ megapixels/cli_datasets.py | 36 ++ megapixels/datasets/commands/crop.py | 104 ++++ megapixels/datasets/commands/extract.py | 86 ++++ megapixels/datasets/commands/face.py | 117 +++++ megapixels/datasets/commands/resize.py | 81 +++ notes.md | 70 +++ notes/cheat-sheets/bash-cheat-sheet.sh | 410 +++++++++++++++ notes/cheat-sheets/docker-cheat-sheet.md | 795 +++++++++++++++++++++++++++++ notes/cheat-sheets/tmux-cheat-sheet.md | 188 +++++++ notes/frameworks/darknet.md | 13 + notes/frameworks/docker.md | 28 + notes/frameworks/nvidia.md | 22 + notes/frameworks/progress_gan.md | 15 + notes/utils/image_utils.md | 302 +++++++++++ 31 files changed, 4258 insertions(+) create mode 100644 README.md create mode 100644 environment.yml create mode 100644 megapixels/admin/commands/rsync.py create mode 100644 megapixels/app/models/__init__.py create mode 100644 megapixels/app/models/bbox.py create mode 100644 megapixels/app/models/click_factory.py create mode 100644 megapixels/app/processors/__init__.py create mode 100644 megapixels/app/processors/face_detector.py create mode 100644 megapixels/app/settings/__init__.py create mode 100644 megapixels/app/settings/app_cfg.py create mode 100644 megapixels/app/settings/types.py create mode 100644 megapixels/app/utils/__init__.py create mode 100644 megapixels/app/utils/click_utils.py create mode 100644 megapixels/app/utils/file_utils.py create mode 100644 megapixels/app/utils/im_utils.py create mode 100644 megapixels/app/utils/logger_utils.py create mode 100644 megapixels/cli_admin.py create mode 100644 megapixels/cli_datasets.py create mode 100644 megapixels/datasets/commands/crop.py create mode 100644 megapixels/datasets/commands/extract.py create mode 100644 megapixels/datasets/commands/face.py create mode 100644 megapixels/datasets/commands/resize.py create mode 100644 notes.md create mode 100644 notes/cheat-sheets/bash-cheat-sheet.sh create mode 100644 notes/cheat-sheets/docker-cheat-sheet.md create mode 100644 notes/cheat-sheets/tmux-cheat-sheet.md create mode 100644 notes/frameworks/darknet.md create mode 100644 notes/frameworks/docker.md create mode 100644 notes/frameworks/nvidia.md create mode 100644 notes/frameworks/progress_gan.md create mode 100644 notes/utils/image_utils.md diff --git a/README.md b/README.md new file mode 100644 index 00000000..3474382d --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# MegaPixels + +FaceQuery.me, mozilla, nytimes \ No newline at end of file diff --git a/environment.yml b/environment.yml new file mode 100644 index 00000000..5f608f80 --- /dev/null +++ b/environment.yml @@ -0,0 +1,207 @@ +name: megapixels +channels: + - pytorch + - conda-forge + - alexbw + - defaults +dependencies: + - lua=5.3.2=1 + - lua-cwrap=0.1=lua5.3_2 + - lua-paths=0.1=lua5.3_1 + - lua-torch=7.0.3=lua5.3_0 + - luarocks=2.2.1=lua5.3_0 + - ca-certificates=2018.4.16=0 + - mpi=1.0=openmpi + - openmpi=3.1.0=h26a2512_3 + - _nb_ext_conf=0.4.0=py36_1 + - anaconda-client=1.6.14=py36_0 + - asn1crypto=0.24.0=py36_0 + - backcall=0.1.0=py36_0 + - blas=1.0=mkl + - bleach=2.1.3=py36_0 + - bzip2=1.0.6=h14c3975_5 + - cairo=1.14.12=h7636065_2 + - certifi=2018.4.16=py36_0 + - cffi=1.11.5=py36h9745a5d_0 + - chardet=3.0.4=py36h0f667ec_1 + - clyent=1.2.2=py36h7e57e65_1 + - cryptography=2.2.2=py36h14c3975_0 + - cudatoolkit=9.0=h13b8566_0 + - decorator=4.3.0=py36_0 + - entrypoints=0.2.3=py36h1aec115_2 + - ffmpeg=4.0=h04d0a96_0 + - fontconfig=2.12.6=h49f89f6_0 + - freetype=2.8=hab7d2ae_1 + - glib=2.56.1=h000015b_0 + - gmp=6.1.2=h6c8ec71_1 + - graphite2=1.3.11=h16798f4_2 + - harfbuzz=1.7.6=h5f0a787_1 + - hdf5=1.10.2=hba1933b_1 + - html5lib=1.0.1=py36h2f9c1c0_0 + - icu=58.2=h9c2bf20_1 + - idna=2.7=py36_0 + - intel-openmp=2018.0.3=0 + - ipykernel=4.8.2=py36_0 + - ipython=6.4.0=py36_0 + - ipython_genutils=0.2.0=py36hb52b0d5_0 + - ipywidgets=7.2.1=py36_0 + - jasper=1.900.1=hd497a04_4 + - jedi=0.12.0=py36_1 + - jinja2=2.10=py36ha16c418_0 + - jpeg=9b=h024ee3a_2 + - jsonschema=2.6.0=py36h006f8b5_0 + - jupyter_client=5.2.3=py36_0 + - jupyter_core=4.4.0=py36h7c827e3_0 + - libedit=3.1.20170329=h6b74fdf_2 + - libffi=3.2.1=hd88cf55_4 + - libgcc-ng=7.2.0=hdf63c60_3 + - libgfortran=3.0.0=1 + - libgfortran-ng=7.2.0=hdf63c60_3 + - libopencv=3.4.1=h1a3b859_1 + - libopus=1.2.1=hb9ed12e_0 + - libpng=1.6.34=hb9fc6fc_0 + - libprotobuf=3.5.2=h6f1eeef_0 + - libsodium=1.0.16=h1bed415_0 + - libstdcxx-ng=7.2.0=hdf63c60_3 + - libtiff=4.0.9=he85c1e1_1 + - libvpx=1.7.0=h439df22_0 + - libxcb=1.13=h1bed415_1 + - libxml2=2.9.8=h26e45fe_1 + - markupsafe=1.0=py36hd9260cd_1 + - mistune=0.8.3=py36h14c3975_1 + - mkl=2018.0.3=1 + - mkl-rt=11.1=p0 + - mkl_fft=1.0.1=py36h3010b51_0 + - mkl_random=1.0.1=py36h629b387_0 + - nb_anacondacloud=1.4.0=py36_0 + - nb_conda=2.2.1=py36h8118bb2_0 + - nb_conda_kernels=2.1.0=py36_0 + - nbconvert=5.3.1=py36hb41ffb7_0 + - nbformat=4.4.0=py36h31c9010_0 + - nbpresent=3.0.2=py36h5f95a39_1 + - ncurses=6.1=hf484d3e_0 + - ninja=1.8.2=py36h6bb024c_1 + - notebook=5.5.0=py36_0 + - numpy=1.14.5=py36hcd700cb_3 + - numpy-base=1.14.5=py36hdbf6ddf_3 + - olefile=0.45.1=py36_0 + - opencv=3.4.1=py36h6fd60c2_2 + - openssl=1.0.2o=h20670df_0 + - pandoc=2.2.1=h629c226_0 + - pandocfilters=1.4.2=py36ha6701b7_1 + - parso=0.2.1=py36_0 + - pcre=8.42=h439df22_0 + - pexpect=4.6.0=py36_0 + - pickleshare=0.7.4=py36h63277f8_0 + - pillow=5.1.0=py36h3deb7b8_0 + - pip=10.0.1=py36_0 + - pixman=0.34.0=hceecf20_3 + - prompt_toolkit=1.0.15=py36h17d85b1_0 + - ptyprocess=0.6.0=py36_0 + - py-opencv=3.4.1=py36h0676e08_1 + - pycparser=2.18=py36hf9f622e_1 + - pygments=2.2.0=py36h0d3125c_0 + - pyopenssl=18.0.0=py36_0 + - pysocks=1.6.8=py36_0 + - python=3.6.6=hc3d631a_0 + - python-dateutil=2.7.3=py36_0 + - pytz=2018.5=py36_0 + - pyyaml=3.12=py36hafb9ca4_1 + - pyzmq=17.0.0=py36h14c3975_0 + - readline=7.0=ha6073c6_4 + - requests=2.19.1=py36_0 + - send2trash=1.5.0=py36_0 + - setuptools=39.2.0=py36_0 + - simplegeneric=0.8.1=py36_2 + - six=1.11.0=py36h372c433_1 + - sqlite=3.24.0=h84994c4_0 + - terminado=0.8.1=py36_1 + - testpath=0.3.1=py36h8cadb63_0 + - tk=8.6.7=hc745277_3 + - tornado=5.0.2=py36_0 + - traitlets=4.3.2=py36h674d592_0 + - urllib3=1.23=py36_0 + - wcwidth=0.1.7=py36hdf4376a_0 + - webencodings=0.5.1=py36h800622e_1 + - wheel=0.31.1=py36_0 + - widgetsnbextension=3.2.1=py36_0 + - xz=5.2.4=h14c3975_4 + - yaml=0.1.7=had09818_2 + - zeromq=4.2.5=h439df22_0 + - zlib=1.2.11=ha838bed_2 + - cuda90=1.0=h6433d27_0 + - pytorch=0.4.0=py36_cuda9.0.176_cudnn7.1.2_1 + - torchvision=0.2.1=py36_1 + - pip: + - absl-py==0.2.2 + - astor==0.7.1 + - beautifulsoup4==4.6.0 + - blocksparse==1.0.0 + - bs4==0.0.1 + - cachetools==2.1.0 + - click==6.7 + - cloudpickle==0.5.3 + - cycler==0.10.0 + - cython==0.28.4 + - dask==0.18.1 + - dlib==19.15.0 + - flask==1.0.2 + - flask-cors==3.0.6 + - future==0.16.0 + - gast==0.2.0 + - google-api-core==1.4.1 + - google-auth==1.5.1 + - google-cloud-core==0.28.1 + - google-cloud-storage==1.13.0 + - google-images-download==2.3.0 + - google-resumable-media==0.3.1 + - googleapis-common-protos==1.6.0b6 + - grpcio==1.13.0 + - h5py==2.8.0 + - horovod==0.13.8 + - imagehash==4.0 + - imageio==2.3.0 + - imutils==0.4.6 + - itsdangerous==0.24 + - jonasz-master-thesis==0.1 + - keras==2.2.0 + - keras-applications==1.0.2 + - keras-preprocessing==1.0.1 + - kiwisolver==1.0.1 + - lmdb==0.94 + - markdown==2.6.11 + - matplotlib==2.2.2 + - moviepy==0.2.3.5 + - networkx==2.1 + - opencv-python==3.4.2.17 + - pandas==0.23.3 + - protobuf==3.6.0 + - pyasn1==0.4.4 + - pyasn1-modules==0.2.2 + - pyglet==1.3.2 + - pymediainfo==2.3.0 + - pyopengl==3.1.0 + - pyparsing==2.2.0 + - python-slugify==1.2.5 + - pywavefront==0.3.2 + - pywavelets==0.5.2 + - rsa==4.0 + - scikit-image==0.14.0 + - scikit-learn==0.19.2 + - scipy==1.1.0 + - selenium==3.13.0 + - tensorboard==1.8.0 + - tensorflow-gpu==1.8.0 + - termcolor==1.1.0 + - tflearn==0.3.2 + - toolz==0.9.0 + - toposort==1.5 + - torch==0.4.0 + - tqdm==4.23.4 + - unicode==2.6 + - unidecode==1.0.22 + - visvis==1.11.1 + - werkzeug==0.14.1 + - wikipedia==1.4.0 +prefix: /home/adam/anaconda3/envs/megapixels + diff --git a/megapixels/admin/commands/rsync.py b/megapixels/admin/commands/rsync.py new file mode 100644 index 00000000..a821b460 --- /dev/null +++ b/megapixels/admin/commands/rsync.py @@ -0,0 +1,106 @@ +""" +Parallel rsync media_records between drives +For parallel rsync with media records, use vframe/commands/rsync +""" + +import click + +from app.settings import types +from app.utils import click_utils +from app.settings import app_cfg as cfg + +@click.command() +@click.option('-i', '--input', 'dir_in', required=True, + help='Input directory') +@click.option('-o', '--output', 'dir_out', required=True, + help='Output directory') +@click.option('-t', '--threads', 'opt_threads', default=8, + help='Number of threads') +@click.option('--validate/--no-validate', 'opt_validate', is_flag=True, default=False, + help='Validate files after copy') +@click.option('--extract/--no-extract', 'opt_extract', is_flag=True, default=False, + help='Extract files after copy') +@click.pass_context +def cli(ctx, dir_in, dir_out, opt_threads, opt_validate, opt_extract): + """rsync folders""" + + import os + from os.path import join + from pathlib import Path + + # NB deactivate logger in imported module + import logging + logging.getLogger().addHandler(logging.NullHandler()) + from parallel_sync import rsync + + from app.settings.paths import Paths + from app.utils import logger_utils, file_utils + + # ------------------------------------------------- + # process here + + log = logger_utils.Logger.getLogger() + log.info('RSYNC from {} to {}'.format(dir_in, dir_out)) + log.info('opt_extract: {}'.format(opt_extract)) + log.info('opt_validate: {}'.format(opt_validate)) + log.info('opt_threads: {}'.format(opt_validate)) + + file_utils.mkdirs(dir_out) + + rsync.copy(dir_in, dir_out, parallelism=opt_threads, + validate=opt_validate, extract=opt_extract) + + log.info('done rsyncing') + + + # --------------------------------------------------------------- + + + + # if dir_in: + # # use input filepath as source + # if not Path(dir_in).is_dir(): + # log.error('{} is not a directory'.format(dir_in)) + # ctx.exit() + # if not Path(dir_out).is_dir(): + # ctx.log.error('{} is not a directory'.format(dir_out)) + # return + + # log.info('RSYNC from {} to {}'.format(dir_in, dir_out)) + # log.debug('opt_validate: {}'.format(opt_validate)) + # log.debug('opt_extract: {}'.format(opt_extract)) + # # local_copy(paths, parallelism=10, extract=False, validate=False): + # file_utils.mkdirs(dir_out) + # rsync.copy(dir_in, dir_out, parallelism=opt_threads, + # validate=opt_validate, extract=opt_extract) + # else: + # log.debug('get paths') + # # use source mappings as rsync source + # if not opt_media_format: + # ctx.log.error('--media format not supplied for source mappings') + # return + + # # ensure FILEPATH metadata exists + # # parallel-rsync accepts a list of tupes (src, dst) + # file_routes = [] + # for chair_item in chair_items: + # item = chair_item.item + # sha256 = chair_item.item.sha256 + # filepath_metadata = item.get_metadata(types.Metadata.FILEPATH) + # if not filepath_metadata: + # ctx.log.error('no FILEPATH metadata') + # return + # fp_media = + # src = join('') + # dir_media = Paths.media_dir(opt_media_format, data_store=opt_disk, verified=ctx.opts['verified']) + # dst = join('') + # file_routes.append((src, dst)) + + # ctx.log.debug('dir_media: {}'.format(dir_media)) + # return + + # # ------------------------------------------------- + + # # send back to sink + # for chair_item in chair_items: + # sink.send(chair_item) diff --git a/megapixels/app/models/__init__.py b/megapixels/app/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/megapixels/app/models/bbox.py b/megapixels/app/models/bbox.py new file mode 100644 index 00000000..41b67416 --- /dev/null +++ b/megapixels/app/models/bbox.py @@ -0,0 +1,236 @@ +from dlib import rectangle as dlib_rectangle +import numpy as np + +class BBoxPoint: + + def __init__(self, x, y): + self._x = x + self._y = y + + @property + def x(self): + return self._x + + @property + def y(self): + return self._y + + def offset(self, x, y): + return (self._x + x, self._y + y) + + def tuple(self): + return (self._x, self._y) + + +class BBox: + + def __init__(self, x1, y1, x2, y2): + """Represents a bounding box and provides methods for accessing and modifying + :param x1: normalized left coord + :param y1: normalized top coord + :param x2: normalized right coord + :param y2: normalized bottom coord + """ + self._x1 = x1 + self._y1 = y1 + self._x2 = x2 + self._y2 = y2 + self._width = x2 - x1 + self._height = y2 - y1 + self._cx = x1 + (self._width // 2) + self._cy = y1 + (self._height // 2) + self._tl = (x1, y1) + self._br = (x2, y2) + self._rect = (self._x1, self._y1, self._x2, self._y2) + + + @property + def pt_tl(self): + return self._tl + + @property + def pt_br(self): + return self._br + + @property + def x(self): + return self._x1 + + @property + def y(self): + return self._y1 + + @property + def x1(self): + return self._x1 + + @property + def y1(self): + return self._y1 + + + @property + def x2(self): + return self._x2 + + @property + def y2(self): + return self._y2 + + @property + def height(self): + return self._height + + @property + def width(self): + return self._width + + @property + def h(self): + return self._height + + @property + def w(self): + return self._width + + @property + def cx(self): + return self._cx + + @property + def cy(self): + return self._cy + + # # ----------------------------------------------------------------- + # # Utils + + # def constrain(self, dim): + + + # ----------------------------------------------------------------- + # Modify + + def expand_dim(self, amt, dim): + """Expands BBox within dim + :param box: (tuple) left, top, right, bottom + :param dim: (tuple) width, height + :returns (BBox) in pixel dimensions + """ + # expand + rect_exp = list( (np.array(self._rect) + np.array([-amt, -amt, amt, amt])).astype('int')) + # outliers + oob = list(range(4)) + oob[0] = min(rect_exp[0], 0) + oob[1] = min(rect_exp[1], 0) + oob[2] = dim[0] - max(rect_exp[2], 2) + oob[3] = dim[1] - max(rect_exp[3], 3) + oob = np.array(oob) + oob[oob > 0] = 0 + # amount + oob = np.absolute(oob) + # threshold + rect_exp[0] = max(rect_exp[0], 0) + rect_exp[1] = max(rect_exp[1], 0) + rect_exp[2] = min(rect_exp[2], dim[0]) + rect_exp[3] = min(rect_exp[3], dim[1]) + # redistribute oob amounts + oob = np.array([-oob[2], -oob[3], oob[0], oob[1]]) + rect_exp = np.add(np.array(rect_exp), oob) + return BBox(*rect_exp) + + + # ----------------------------------------------------------------- + # Convert to + + def to_dim(self, dim): + """scale is (w, h) is tuple of dimensions""" + w, h = dim + rect = list((np.array(self._rect) * np.array([w, h, w, h])).astype('int')) + return BBox(*rect) + + def normalize(self, rect, dim): + w, h = dim + x1, y1, x2, y2 = rect + return (x1 / w, y1 / h, x2 / w, y2 / h) + + # ----------------------------------------------------------------- + # Format as + + def as_xyxy(self): + """Converts BBox back to x1, y1, x2, y2 rect""" + return (self._x1, self._y1, self._x2, self._y2) + + def as_xywh(self): + """Converts BBox back to haar type""" + return (self._x1, self._y1, self._width, self._height) + + def as_trbl(self): + """Converts BBox to CSS (top, right, bottom, left)""" + return (self._y1, self._x2, self._y2, self._x1) + + def as_dlib(self): + """Converts BBox to dlib rect type""" + return dlib.rectangle(self._x1, self._y1, self._x2, self._y2) + + def as_yolo(self): + """Converts BBox to normalized center x, center y, w, h""" + return (self._cx, self._cy, self._width, self._height) + + + # ----------------------------------------------------------------- + # Create from + + @classmethod + def from_xyxy_dim(cls, x1, y1, x2, y2, dim): + """Converts x1, y1, w, h to BBox and normalizes + :returns BBox + """ + rect = cls.normalize(cls, (x1, y1, x2, y2), dim) + return cls(*rect) + + @classmethod + def from_xywh_dim(cls, x, y, w, h, dim): + """Converts x1, y1, w, h to BBox and normalizes + :param rect: (list) x1, y1, w, h + :param dim: (list) w, h + :returns BBox + """ + rect = cls.normalize(cls, (x, y, x + w, y + h), dim) + return cls(*rect) + + @classmethod + def from_xywh(cls, x, y, w, h): + """Converts x1, y1, w, h to BBox + :param rect: (list) x1, y1, w, h + :param dim: (list) w, h + :returns BBox + """ + return cls(x, y, x+w, y+h) + + @classmethod + def from_css(cls, rect, dim): + """Converts rect from CSS (top, right, bottom, left) to BBox + :param rect: (list) x1, y1, x2, y2 + :param dim: (list) w, h + :returns BBox + """ + rect = (rect[3], rect[0], rect[1], rect[2]) + rect = cls.normalize(cls, rect, dim) + return cls(*rect) + + @classmethod + def from_dlib_dim(cls, rect, dim): + """Converts dlib.rectangle to BBox + :param rect: (list) x1, y1, x2, y2 + :param dim: (list) w, h + :returns dlib.rectangle + """ + rect = (rect.left(), rect.top(), rect.right(), rect.bottom()) + rect = cls.normalize(cls, rect, dim) + return cls(*rect) + + + def str(self): + """Return BBox as a string "x1, y1, x2, y2" """ + return self.as_box() + diff --git a/megapixels/app/models/click_factory.py b/megapixels/app/models/click_factory.py new file mode 100644 index 00000000..61a3b5e5 --- /dev/null +++ b/megapixels/app/models/click_factory.py @@ -0,0 +1,145 @@ +""" +Click processor factory +- Inspired by and used code from @wiretapped's HTSLAM codebase +- In particular the very useful +""" + +import os +import sys +from os.path import join +from pathlib import Path +import os +from os.path import join +import sys +from functools import update_wrapper, wraps +import itertools +from pathlib import Path +from glob import glob +import importlib +import logging + +import click +from app.settings import app_cfg as cfg + + +# -------------------------------------------------------- +# Click Group Class +# -------------------------------------------------------- + +# set global variable during parent class create +dir_plugins = None # set in create + +class ClickComplex: + """Wrapper generator for custom Click CLI's based on LR's coroutine""" + + def __init__(self): + pass + + + class CustomGroup(click.Group): + #global dir_plugins # from CliGenerator init + + # lists commands in plugin directory + def list_commands(self, ctx): + global dir_plugins # from CliGenerator init + rv = list(self.commands.keys()) + fp_cmds = [Path(x) for x in Path(dir_plugins).iterdir() \ + if str(x).endswith('.py') \ + and '__init__' not in str(x)] + for fp_cmd in fp_cmds: + try: + assert fp_cmd.name not in rv, "[-] Error: {} can't exist in cli.py and {}".format(fp_cmd.name) + except Exception as ex: + logging.getLogger('app').error('{}'.format(ex)) + rv.append(fp_cmd.stem) + rv.sort() + return rv + + # Complex version: gets commands in directory and in this file + # Based on code from @wiretapped + HTSLAM + def get_command(self, ctx, cmd_name): + global dir_plugins + if cmd_name in self.commands: + return self.commands[cmd_name] + ns = {} + fpp_cmd = Path(dir_plugins, cmd_name + '.py') + fp_cmd = fpp_cmd.as_posix() + if not fpp_cmd.exists(): + sys.exit('[-] {} file does not exist'.format(fpp_cmd)) + code = compile(fpp_cmd.read_bytes(), fp_cmd, 'exec') + try: + eval(code, ns, ns) + except Exception as ex: + logging.getLogger('vframe').error('exception: {}'.format(ex)) + @click.command() + def _fail(): + raise Exception('while loading {}'.format(fpp_cmd.name)) + _fail.short_help = repr(ex) + _fail.help = repr(ex) + return _fail + if 'cli' not in ns: + sys.exit('[-] Error: {} does not contain a cli function'.format(fp_cmd)) + return ns['cli'] + + @classmethod + def create(self, dir_plugins_local): + global dir_plugins + dir_plugins = dir_plugins_local + return self.CustomGroup + + + +class ClickSimple: + """Wrapper generator for custom Click CLI's""" + + def __init__(self): + pass + + + class CustomGroup(click.Group): + #global dir_plugins # from CliGenerator init + + # lists commands in plugin directory + def list_commands(self, ctx): + global dir_plugins # from CliGenerator init + rv = list(self.commands.keys()) + fp_cmds = [Path(x) for x in Path(dir_plugins).iterdir() \ + if str(x).endswith('.py') \ + and '__init__' not in str(x)] + for fp_cmd in fp_cmds: + assert fp_cmd.name not in rv, "[-] Error: {} can't exist in cli.py and {}".format(fp_cmd.name) + rv.append(fp_cmd.stem) + rv.sort() + return rv + + # Complex version: gets commands in directory and in this file + # from HTSLAM + def get_command(self, ctx, cmd_name): + global dir_plugins # from CliGenerator init + if cmd_name in self.commands: + return self.commands[cmd_name] + ns = {} + fpp_cmd = Path(dir_plugins, cmd_name + '.py') + fp_cmd = fpp_cmd.as_posix() + if not fpp_cmd.exists(): + sys.exit('[-] {} file does not exist'.format(fpp_cmd)) + code = compile(fpp_cmd.read_bytes(), fp_cmd, 'exec') + try: + eval(code, ns, ns) + except Exception as ex: + logging.getLogger('vframe').error('exception: {}'.format(ex)) + @click.command() + def _fail(): + raise Exception('while loading {}'.format(fpp_cmd.name)) + _fail.short_help = repr(ex) + _fail.help = repr(ex) + return _fail + if 'cli' not in ns: + sys.exit('[-] Error: {} does not contain a cli function'.format(fp_cmd)) + return ns['cli'] + + @classmethod + def create(self, dir_plugins_local): + global dir_plugins + dir_plugins = dir_plugins_local + return self.CustomGroup diff --git a/megapixels/app/processors/__init__.py b/megapixels/app/processors/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/megapixels/app/processors/face_detector.py b/megapixels/app/processors/face_detector.py new file mode 100644 index 00000000..02d068dc --- /dev/null +++ b/megapixels/app/processors/face_detector.py @@ -0,0 +1,103 @@ +import os +from os.path import join +from pathlib import Path + +import cv2 as cv +import numpy as np +import dlib +# import imutils + +from app.utils import im_utils, logger_utils +from app.models.bbox import BBox +from app.settings import app_cfg as cfg + +class DetectorDLIBCNN: + + dnn_size = (300, 300) + pyramids = 0 + conf_thresh = 0.85 + + def __init__(self, opt_gpu): + self.log = logger_utils.Logger.getLogger() + cuda_visible_devices = os.getenv('CUDA_VISIBLE_DEVICES', '') + os.environ['CUDA_VISIBLE_DEVICES'] = str(opt_gpu) + self.log.info('load model: {}'.format(cfg.DIR_MODELS_DLIB_CNN)) + self.detector = dlib.cnn_face_detection_model_v1(cfg.DIR_MODELS_DLIB_CNN) + os.environ['CUDA_VISIBLE_DEVICES'] = cuda_visible_devices # reset + + def detect(self, im, opt_size=None, opt_conf_thresh=None, opt_pyramids=None): + rois = [] + conf_thresh = self.conf_thresh if opt_conf_thresh is None else opt_conf_thresh + pyramids = self.pyramids if opt_pyramids is None else opt_pyramids + dnn_size = self.dnn_size if opt_size is None else opt_size + # resize image + im = im_utils.resize(im, width=dnn_size[0], height=dnn_size[1]) + dim = im.shape[:2][::-1] + im = im_utils.bgr2rgb(im) # convert to RGB for dlib + # run detector + mmod_rects = self.detector(im, 1) + # sort results + for mmod_rect in mmod_rects: + if mmod_rect.confidence > conf_thresh: + bbox = BBox.from_dlib_dim(mmod_rect.rect, dim) + rois.append(bbox) + return rois + + +class DetectorDLIBHOG: + + size = (320, 240) + pyramids = 0 + + def __init__(self): + self.detector = dlib.get_frontal_face_detector() + + def detect(self, im, opt_size=None, opt_conf_thresh=None, opt_pyramids=0): + conf_thresh = self.conf_thresh if opt_conf_thresh is None else opt_conf_thresh + dnn_size = self.size if opt_size is None else opt_size + pyramids = self.pyramids if opt_pyramids is None else opt_pyramids + + im = im_utils.resize(im, width=opt_size[0], height=opt_size[1]) + dim = im.shape[:2][::-1] + im = im_utils.bgr2rgb(im) # ? + hog_results = self.detector.run(im, pyramids) + + rois = [] + if len(hog_results[0]) > 0: + for rect, score, direction in zip(*hog_results): + if score > opt_conf_thresh: + bbox = BBox.from_dlib_dim(rect, dim) + rois.append(bbox) + return rois + +class DetectorCVDNN: + + dnn_scale = 1.0 # fixed + dnn_mean = (104.0, 177.0, 123.0) # fixed + dnn_crop = False # crop or force resize + size = (300, 300) + conf_thresh = 0.85 + + def __init__(self): + fp_prototxt = join(cfg.DIR_MODELS_CAFFE, 'face_detect', 'opencv_face_detector.prototxt') + fp_model = join(cfg.DIR_MODELS_CAFFE, 'face_detect', 'opencv_face_detector.caffemodel') + self.net = cv.dnn.readNet(fp_prototxt, fp_model) + self.net.setPreferableBackend(cv.dnn.DNN_BACKEND_OPENCV) + self.net.setPreferableTarget(cv.dnn.DNN_TARGET_CPU) + + def detect(self, im, opt_size=None, opt_conf_thresh=None): + """Detects faces and returns (list) of (BBox)""" + conf_thresh = self.conf_thresh if opt_conf_thresh is None else opt_conf_thresh + dnn_size = self.size if opt_size is None else opt_size + im = cv.resize(im, dnn_size) + blob = cv.dnn.blobFromImage(im, self.dnn_scale, dnn_size, self.dnn_mean) + self.net.setInput(blob) + net_outputs = self.net.forward() + + rois = [] + for i in range(0, net_outputs.shape[2]): + conf = net_outputs[0, 0, i, 2] + if conf > opt_conf_thresh: + rect_norm = net_outputs[0, 0, i, 3:7] + rois.append(BBox(*rect_norm)) + return rois \ No newline at end of file diff --git a/megapixels/app/settings/__init__.py b/megapixels/app/settings/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/megapixels/app/settings/app_cfg.py b/megapixels/app/settings/app_cfg.py new file mode 100644 index 00000000..739ddce2 --- /dev/null +++ b/megapixels/app/settings/app_cfg.py @@ -0,0 +1,90 @@ +import os +from os.path import join +import logging +import collections + +import cv2 as cv + +from app.settings import types +from app.utils import click_utils + + +# ----------------------------------------------------------------------------- +# Enun lists used for custom Click Params +# ----------------------------------------------------------------------------- + +FaceDetectNetVar = click_utils.ParamVar(types.FaceDetectNet) + +LogLevelVar = click_utils.ParamVar(types.LogLevel) + +# # data_store +DATA_STORE = '/data_store_hdd/' +DIR_DATASETS = join(DATA_STORE,'datasets') +DIR_APPS = join(DATA_STORE,'apps') +DIR_APP = join(DIR_APPS,'megapixels') +DIR_MODELS = join(DIR_APP,'models') + +# # Frameworks +DIR_MODELS_CAFFE = join(DIR_MODELS,'caffe') +DIR_MODELS_DARKNET = join(DIR_MODELS,'darknet') +DIR_MODELS_DARKNET_PJREDDIE = join(DIR_MODELS_DARKNET, 'pjreddie') +DIR_MODELS_PYTORCH = join(DIR_MODELS,'pytorch') +DIR_MODELS_TORCH = join(DIR_MODELS,'torch') +DIR_MODELS_MXNET = join(DIR_MODELS,'mxnet') +DIR_MODELS_TF = join(DIR_MODELS,'tensorflow') +DIR_MODELS_DLIB = join(DIR_MODELS,'dlib') +DIR_MODELS_DLIB_CNN = join(DIR_MODELS_DLIB, 'mmod_human_face_detector.dat') +DIR_MODELS_DLIB_5PT = join(DIR_MODELS_DLIB, 'shape_predictor_5_face_landmarks.dat') +DIR_MODELS_DLIB_68PT = join(DIR_MODELS_DLIB, 'shape_predictor_68_face_landmarks.dat') + + +# Test images +DIR_TEST_IMAGES = join(DIR_APP, 'test', 'images') + +# ----------------------------------------------------------------------------- +# Drawing, GUI settings +# ----------------------------------------------------------------------------- +DIR_ASSETS = join(DIR_APP, 'assets') +FP_FONT = join(DIR_ASSETS, 'font') + + +# ----------------------------------------------------------------------------- +# click chair settings +# ----------------------------------------------------------------------------- +DIR_COMMANDS_PROCESSOR_ADMIN = 'admin/commands' +DIR_COMMANDS_PROCESSOR_DATASETS = 'datasets/commands' + +# ----------------------------------------------------------------------------- +# Filesystem settings +# hash trees enforce a maximum number of directories per directory +# ----------------------------------------------------------------------------- +ZERO_PADDING = 6 # padding for enumerated image filenames +#FRAME_NAME_ZERO_PADDING = 6 # is this active?? +CKPT_ZERO_PADDING = 9 +HASH_TREE_DEPTH = 3 +HASH_BRANCH_SIZE = 3 + +# ----------------------------------------------------------------------------- +# Logging options exposed for custom click Params +# ----------------------------------------------------------------------------- +LOGGER_NAME = 'app' +LOGLEVELS = { + types.LogLevel.DEBUG: logging.DEBUG, + types.LogLevel.INFO: logging.INFO, + types.LogLevel.WARN: logging.WARN, + types.LogLevel.ERROR: logging.ERROR, + types.LogLevel.CRITICAL: logging.CRITICAL +} +LOGLEVEL_OPT_DEFAULT = types.LogLevel.DEBUG.name +#LOGFILE_FORMAT = "%(asctime)s: %(levelname)s: %(message)s" +#LOGFILE_FORMAT = "%(levelname)s:%(name)s: %(message)s" +#LOGFILE_FORMAT = "%(levelname)s: %(message)s" +#LOGFILE_FORMAT = "%(filename)s:%(lineno)s %(funcName)s() %(message)s" +# colored logs +""" +black, red, green, yellow, blue, purple, cyan and white. +{color}, fg_{color}, bg_{color}: Foreground and background colors. +bold, bold_{color}, fg_bold_{color}, bg_bold_{color}: Bold/bright colors. +reset: Clear all formatting (both foreground and background colors). +""" +LOGFILE_FORMAT = "%(log_color)s%(levelname)-8s%(reset)s %(cyan)s%(filename)s:%(lineno)s:%(bold_cyan)s%(funcName)s() %(reset)s%(message)s" \ No newline at end of file diff --git a/megapixels/app/settings/types.py b/megapixels/app/settings/types.py new file mode 100644 index 00000000..0c3d7942 --- /dev/null +++ b/megapixels/app/settings/types.py @@ -0,0 +1,29 @@ +from enum import Enum + +def find_type(name, enum_type): + for enum_opt in enum_type: + if name == enum_opt.name.lower(): + return enum_opt + return None + + + +class FaceDetectNet(Enum): + """Scene text detector networks""" + HAAR, DLIB_CNN, DLIB_HOG, CVDNN = range(4) + +class CVBackend(Enum): + """OpenCV 3.4.2+ DNN target type""" + DEFAULT, HALIDE, INFER_ENGINE, OPENCV = range(4) + +class CVTarget(Enum): + """OpenCV 3.4.2+ DNN backend processor type""" + CPU, OPENCL, OPENCL_FP16, MYRIAD = range(4) + +# --------------------------------------------------------------------- +# Logger, monitoring +# -------------------------------------------------------------------- + +class LogLevel(Enum): + """Loger vebosity""" + DEBUG, INFO, WARN, ERROR, CRITICAL = range(5) diff --git a/megapixels/app/utils/__init__.py b/megapixels/app/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/megapixels/app/utils/click_utils.py b/megapixels/app/utils/click_utils.py new file mode 100644 index 00000000..dc00f58c --- /dev/null +++ b/megapixels/app/utils/click_utils.py @@ -0,0 +1,62 @@ +""" +Custom Click parameter types +""" +import click + +from app.settings import app_cfg as cfg +from app.settings import types + + +# -------------------------------------------------------- +# Click command helpers +# -------------------------------------------------------- +def enum_to_names(enum_type): + return {x.name.lower(): x for x in enum_type} + +def show_help(enum_type): + names = enum_to_names(enum_type) + return 'Options: "{}"'.format(', '.join(list(names.keys()))) + +def get_default(opt): + return opt.name.lower() + + +# -------------------------------------------------------- +# Custom Click parameter class +# -------------------------------------------------------- + + +class ParamVar(click.ParamType): + + name = 'default_type' + + def __init__(self, param_type): + # self.name = '{}'.format(param_type.name.lower()) + # sealf. + self.ops = {x.name.lower(): x for x in param_type} + + def convert(self, value, param, ctx): + """converts (str) repr to Enum hash""" + try: + return self.ops[value.lower()] + except: + self.fail('{} is not a valid option'.format(value, param, ctx)) + + + + + + + + + + + + + + + + + + + diff --git a/megapixels/app/utils/file_utils.py b/megapixels/app/utils/file_utils.py new file mode 100644 index 00000000..773667b1 --- /dev/null +++ b/megapixels/app/utils/file_utils.py @@ -0,0 +1,400 @@ +""" +File utilities +""" +import sys +import os +from os.path import join +import stat + +from glob import glob +from pprint import pprint +import shutil +import distutils +import pathlib +from pathlib import Path +import json +import csv +import pickle +import threading +from queue import Queue +import time +import logging +import itertools +import collections + +import hashlib +import pymediainfo +import click +from tqdm import tqdm +import cv2 as cv +from PIL import Image +import imutils + +from app.settings import app_cfg as cfg +from app.settings import types + +log = logging.getLogger(cfg.LOGGER_NAME) + + +# ------------------------------------------ +# File I/O read/write little helpers +# ------------------------------------------ + +def glob_multi(dir_in, exts): + files = [] + for e in exts: + files.append(glob(join(dir_in, '*.{}'.format(e)))) + return files + + +def zpad(x, zeros=cfg.ZERO_PADDING): + return str(x).zfill(zeros) + +def get_ext(fpp, lower=True): + """Retuns the file extension w/o dot + :param fpp: (Pathlib.path) filepath + :param lower: (bool) force lowercase + :returns: (str) file extension (ie 'jpg') + """ + fpp = ensure_posixpath(fpp) + ext = fpp.suffix.replace('.', '') + return ext.lower() if lower else ext + + +def convert(fp_in, fp_out): + """Converts between JSON and Pickle formats + Pickle files are about 30-40% smaller filesize + """ + if get_ext(fp_in) == get_ext(fp_out): + log.error('Input: {} and output: {} are the same. Use this to convert.') + + lazywrite(lazyload(fp_in), fp_out) + + +def load_csv(fp_in, as_list=True): + """Loads CSV and retuns list of items + :param fp_in: string filepath to CSV + :returns: list of all CSV data + """ + if not Path(fp_in).exists(): + log.info('loading {}'.format(fp_in)) + log.info('loading: {}'.format(fp_in)) + with open(fp_in, 'r') as fp: + items = csv.DictReader(fp) + if as_list: + items = [x for x in items] + log.info('returning {:,} items'.format(len(items))) + return items + + +def lazywrite(data, fp_out, sort_keys=True): + """Writes JSON or Pickle data""" + ext = get_ext(fp_out) + if ext == 'json': + return write_json(data, fp_out, sort_keys=sort_keys) + elif ext == 'pkl': + return write_pickle(data, fp_out) + else: + raise NotImplementedError('[!] {} is not yet supported. Use .pkl or .json'.format(ext)) + + +def lazyload(fp_in, ordered=True): + """Loads JSON or Pickle serialized data""" + if not Path(fp_in).exists(): + log.error('file does not exist: {}'.format(fp_in)) + return {} + ext = get_ext(fp_in) + if ext == 'json': + items = load_json(fp_in) + elif ext == 'pkl': + items = load_pickle(fp_in) + else: + raise NotImplementedError('[!] {} is not yet supported. Use .pkl or .json'.format(ext)) + + if ordered: + return collections.OrderedDict(sorted(items.items(), key=lambda t: t[0])) + else: + return items + + +def load_text(fp_in): + with open(fp_in, 'rt') as fp: + lines = fp.read().rstrip('\n').split('\n') + return lines + +def load_json(fp_in): + """Loads JSON and returns items + :param fp_in: (str) filepath + :returns: data from JSON + """ + if not Path(fp_in).exists(): + log.error('file does not exist: {}'.format(fp_in)) + return {} + with open(str(fp_in), 'r') as fp: + data = json.load(fp) + return data + + +def load_pickle(fp_in): + """Loads Pickle and returns items + :param fp_in: (str) filepath + :returns: data from JSON + """ + if not Path(fp_in).exists(): + log.error('file does not exist: {}'.format(fp_in)) + return {} + with open(str(fp_in), 'rb') as fp: + data = pickle.load(fp) + return data + + +def order_items(records): + """Orders records by ASC SHA256""" + return collections.OrderedDict(sorted(records.items(), key=lambda t: t[0])) + +def write_text(data, fp_out, ensure_path=True): + if not data: + log.error('no data') + return + + if ensure_path: + mkdirs(fp_out) + with open(fp_out, 'w') as fp: + if type(data) == list: + fp.write('\n'.join(data)) + else: + fp.write(data) + + +def write_pickle(data, fp_out, ensure_path=True): + """ + """ + if ensure_path: + mkdirs(fp_out) # mkdir + with open(fp_out, 'wb') as fp: + pickle.dump(data, fp) + + +def write_json(data, fp_out, minify=True, ensure_path=True, sort_keys=True): + """ + """ + if ensure_path: + mkdirs(fp_out) + with open(fp_out, 'w') as fp: + if minify: + json.dump(data, fp, separators=(',',':'), sort_keys=sort_keys) + else: + json.dump(data, fp, indent=2, sort_keys=sort_keys) + +def write_csv(data, fp_out, header=None): + """ """ + with open(fp_out, 'w') as fp: + writer = csv.DictWriter(fp, fieldnames=header) + writer.writeheader() + if type(data) is dict: + for k, v in data.items(): + fp.writerow('{},{}'.format(k, v)) + + +def write_serialized_items(items, fp_out, ensure_path=True, minify=True, sort_keys=True): + """Writes serialized data + :param items: (dict) a sha256 dict of MappingItems + :param serialize: (bool) serialize the data + :param ensure_path: ensure the parent directories exist + :param minify: reduces JSON file size + """ + log.info('Writing serialized data...') + fpp_out = ensure_posixpath(fp_out) + serialized_items = {k: v.serialize() for k, v in tqdm(items.items()) } + # write data + ext = get_ext(fpp_out) + if ext == 'json': + write_json(serialized_items, fp_out, ensure_path=ensure_path, minify=minify, sort_keys=sort_keys) + elif ext == 'pkl': + write_pickle(serialized_items, fp_out) + else: + raise NotImplementedError('[!] {} is not yet supported. Use .pkl or .json'.format(ext)) + log.info('Wrote {:,} items to {}'.format(len(items), fp_out)) + + +def write_modeled_data(data, fp_out, ensure_path=False): + """ + """ + fpp_out = ensure_posixpath(fp_out) + if ensure_path: + mkdirs(fpp_out) + ext = get_ext(fpp_out) + if ext == 'pkl': + write_pickle(data, str(fp_out)) + else: + raise NotImplementedError('[!] {} is not yet supported. Use .pkl or .json'.format(ext)) + + +# --------------------------------------------------------------------- +# Filepath utilities +# --------------------------------------------------------------------- + +def ensure_posixpath(fp): + """Ensures filepath is pathlib.Path + :param fp: a (str, LazyFile, PosixPath) + :returns: a PosixPath filepath object + """ + if type(fp) == str: + fpp = Path(fp) + elif type(fp) == click.utils.LazyFile: + fpp = Path(fp.name) + elif type(fp) == pathlib.PosixPath: + fpp = fp + else: + raise TypeError('{} is not a valid filepath type'.format(type(fp))) + return fpp + + +def mkdirs(fp): + """Ensure parent directories exist for a filepath + :param fp: string, Path, or click.File + """ + fpp = ensure_posixpath(fp) + fpp = fpp.parent if fpp.suffix else fpp + fpp.mkdir(parents=True, exist_ok=True) + + +def ext_media_format(ext): + """Converts file extension into Enum MediaType + param ext: str of file extension" + """ + for media_format, exts in cfg.VALID_MEDIA_EXTS.items(): + if ext in exts: + return media_format + raise ValueError('{} is not a valid option'.format(ext)) + + +def sha256(fp_in, block_size=65536): + """Generates SHA256 hash for a file + :param fp_in: (str) filepath + :param block_size: (int) byte size of block + :returns: (str) hash + """ + sha256 = hashlib.sha256() + with open(fp_in, 'rb') as fp: + for block in iter(lambda: f.read(block_size), b''): + sha256.update(block) + return sha256.hexdigest() + + +def sha256_tree(sha256): + """Split hash into branches with tree-depth for faster file indexing + :param sha256: str of a sha256 hash + :returns: str with sha256 tree with '/' delimeter + """ + branch_size = cfg.HASH_BRANCH_SIZE + tree_size = cfg.HASH_TREE_DEPTH * branch_size + sha256_tree = [sha256[i:(i+branch_size)] for i in range(0, tree_size, branch_size)] + return '/'.join(sha256_tree) + + +def migrate(fmaps, threads=1, action='copy', force=False): + """Copy/move/symlink files form src to dst directory + :param fmaps: (dict) with 'src' and 'dst' filepaths + :param threads: (int) number of threads + :param action: (str) copy/move/symlink + :param force: (bool) force overwrite existing files + """ + log = log + num_items = len(fmaps) + + def copytree(src, dst, symlinks = False, ignore = None): + # ozxyqk: https://stackoverflow.com/questions/22588225/how-do-you-merge-two-directories-or-move-with-replace-from-the-windows-command + if not os.path.exists(dst): + mkdirs(dst) + # os.makedirs(dst) + shutil.copystat(src, dst) + lst = os.listdir(src) + if ignore: + excl = ignore(src, lst) + lst = [x for x in lst if x not in excl] + for item in lst: + s = os.path.join(src, item) + d = os.path.join(dst, item) + if symlinks and os.path.islink(s): + if os.path.exists(d): + os.remove(d) + os.symlink(os.readlink(s), d) + try: + st = os.lstat(s) + mode = stat.S_IMODE(st.st_mode) + os.lchmod(d, mode) + except: + pass # lchmod not available + elif os.path.isdir(s): + copytree(s, d, symlinks, ignore) + else: + shutil.copy(s, d) + + assert(action in ['copy','move','symlink']) + + if threads > 1: + # threaded + task_queue = Queue() + print_lock = threading.Lock() + + def migrate_action(fmap): + data_local = threading.local() + data_local.src, data_local.dst = (fmap['src'], fmap['dst']) + data_local.src_path = Path(data_local.src) + data_local.dst_path = Path(data_local.dst) + + if force or not data_local.dst_path.exists(): + if action == 'copy': + shutil.copy(data_local.src, data_local.dst) + #if data_local.src_path.is_dir(): + # copytree(data_local.src, data_local.dst) + #else: + elif action == 'move': + shutil.move(data_local.src, data_local.dst) + elif action == 'symlink': + if force: + data_local.dst_path.unlink() + Path(data_local.src).symlink_to(data_local.dst) + + def process_queue(num_items): + # TODO: progress bar + while True: + fmap = task_queue.get() + migrate_action(fmap) + log.info('migrate: {:.2f} {:,}/{:,}'.format( + (task_queue.qsize() / num_items)*100, task_queue.qsize(), num_items)) + task_queue.task_done() + + # avoid race conditions by creating dir structure here + log.info('create directory structure') + for fmap in tqdm(fmaps): + mkdirs(fmap['dst']) + + # init threads + for i in range(threads): + t = threading.Thread(target=process_queue, args=(num_items,)) + t.daemon = True + t.start() + + # process threads + start = time.time() + for fmap in fmaps: + task_queue.put(fmap) + + task_queue.join() + + else: + # non-threaded + for fmap in tqdm(fmaps): + mkdirs(fmap['dst']) + if action == 'copy': + shutil.copy(fmap['src'], fmap['dst']) + elif action == 'move': + shutil.move(fmap['src'], fmap['dst']) + elif action == 'symlink': + if force: + Path(fmap['dst'].unlink()) + Path(fp_src).symlink_to(fp_dst) + return + diff --git a/megapixels/app/utils/im_utils.py b/megapixels/app/utils/im_utils.py new file mode 100644 index 00000000..a0f23cd2 --- /dev/null +++ b/megapixels/app/utils/im_utils.py @@ -0,0 +1,506 @@ +import sys +import os +from os.path import join +import cv2 as cv +import imagehash +from PIL import Image, ImageDraw, ImageFilter, ImageOps +from skimage.filters.rank import entropy +from skimage.morphology import disk +from skimage import feature +# import matplotlib.pyplot as plt +import imutils +import time +import numpy as np +import torch +import torch.nn as nn +import torchvision.models as models +import torchvision.transforms as transforms +from torch.autograd import Variable +from sklearn.metrics.pairwise import cosine_similarity +import datetime + + + + +def compute_features(fe,frames,phashes,phash_thresh=1): + """ + Get vector embedding using FeatureExtractor + :param fe: FeatureExtractor class + :param frames: list of frame images as numpy.ndarray + :param phash_thresh: perceptual hash threshold + :returns: list of feature vectors + """ + vals = [] + phash_pre = phashes[0] + for i,im in enumerate(frames): + if i == 0 or (phashes[i] - phashes[i-1]) > phash_thresh: + vals.append(fe.extract(im)) + else: + vals.append(vals[i-1]) + return vals + + +def ensure_pil(im, bgr2rgb=False): + """Ensure image is Pillow format + :param im: image in numpy or PIL.Image format + :returns: image in Pillow RGB format + """ + try: + im.verify() + return im + except: + if bgr2rgb: + im = cv.cvtColor(im,cv.COLOR_BGR2RGB) + return Image.fromarray(im.astype('uint8'), 'RGB') + +def ensure_np(im): + """Ensure image is Numpy.ndarry format + :param im: image in numpy or PIL.Image format + :returns: image in Numpy uint8 format + """ + if type(im) == np.ndarray: + return im + return np.asarray(im, np.uint8) + + +def resize(im,width=0,height=0): + """resize image using imutils. Use w/h=[0 || None] to prioritize other edge size + :param im: a Numpy.ndarray image + :param wh: a tuple of (width, height) + """ + w = width + h = height + if w is 0 and h is 0: + return im + elif w > 0 and h > 0: + return imutils.resize(im,width=w,height=h) + elif w > 0 and h is 0: + return imutils.resize(im,width=w) + elif w is 0 and h > 0: + return imutils.resize(im,height=h) + else: + return im + +def filter_pixellate(im,num_cells): + """Pixellate image by downsample then upsample + :param im: PIL.Image + :returns: PIL.Image + """ + w,h = im.size + im = im.resize((num_cells,num_cells), Image.NEAREST) + im = im.resize((w,h), Image.NEAREST) + return im + +# Plot images inline using Matplotlib +# def pltimg(im,title=None,mode='rgb',figsize=(8,12),dpi=160,output=None): +# plt.figure(figsize=figsize) +# plt.xticks([]),plt.yticks([]) +# if title is not None: +# plt.title(title) +# if mode.lower() == 'bgr': +# im = cv.cvtColor(im,cv.COLOR_BGR2RGB) + +# f = plt.gcf() +# if mode.lower() =='grey' or mode.lower() == 'gray': +# plt.imshow(im,cmap='gray') +# else: +# plt.imshow(im) +# plt.show() +# plt.draw() +# if output is not None: +# bbox_inches='tight' +# ext=osp.splitext(output)[1].replace('.','') +# f.savefig(output,dpi=dpi,format=ext) +# print('Image saved to: {}'.format(output)) + + + +# Utilities for analyzing frames + +def compute_gray(im): + im = cv.cvtColor(im,cv.COLOR_BGR2GRAY) + n_vals = float(im.shape[0] * im.shape[1]) + avg = np.sum(im[:]) / n_vals + return avg + +def compute_rgb(im): + im = cv.cvtColor(im,cv.COLOR_BGR2RGB) + n_vals = float(im.shape[0] * im.shape[1]) + avg_r = np.sum(im[:,:,0]) / n_vals + avg_g = np.sum(im[:,:,1]) / n_vals + avg_b = np.sum(im[:,:,2]) / n_vals + avg_rgb = np.sum(im[:,:,:]) / (n_vals * 3.0) + return avg_r, avg_b, avg_g, avg_rgb + +def compute_hsv(im): + im = cv.cvtColor(im,cv.COLOR_BGR2HSV) + n_vals = float(im.shape[0] * im.shape[1]) + avg_h = np.sum(frame[:,:,0]) / n_vals + avg_s = np.sum(frame[:,:,1]) / n_vals + avg_v = np.sum(frame[:,:,2]) / n_vals + avg_hsv = np.sum(frame[:,:,:]) / (n_vals * 3.0) + return avg_h, avg_s, avg_v, avg_hsv + +def pys_dhash(im, hashSize=8): + # resize the input image, adding a single column (width) so we + # can compute the horizontal gradient + resized = cv.resize(im, (hashSize + 1, hashSize)) + # compute the (relative) horizontal gradient between adjacent + # column pixels + diff = resized[:, 1:] > resized[:, :-1] + # convert the difference image to a hash + return sum([2 ** i for (i, v) in enumerate(diff.flatten()) if v]) + + +############################################ +# ImageHash +# pip install imagehash +############################################ + + +def compute_ahash(im): + """Compute average hash using ImageHash library + :param im: Numpy.ndarray + :returns: Imagehash.ImageHash + """ + return imagehash.average_hash(ensure_pil(im_pil)) + +def compute_phash(im): + """Compute perceptual hash using ImageHash library + :param im: Numpy.ndarray + :returns: Imagehash.ImageHash + """ + return imagehash.phash(ensure_pil(im)) + +def compute_dhash(im): + """Compute difference hash using ImageHash library + :param im: Numpy.ndarray + :returns: Imagehash.ImageHash + """ + return imagehash.dhash(ensure_pil(im)) + +def compute_whash(im): + """Compute wavelet hash using ImageHash library + :param im: Numpy.ndarray + :returns: Imagehash.ImageHash + """ + return imagehash.whash(ensure_pil(im)) + +def compute_whash_b64(im): + """Compute wavelest hash base64 using ImageHash library + :param im: Numpy.ndarray + :returns: Imagehash.ImageHash + """ + return lambda im: imagehash.whash(ensure_pil(im), mode='db4') + + +############################################ +# Pillow +############################################ + +def sharpen(im): + """Sharpen image using PIL.ImageFilter + param: im: PIL.Image + returns: PIL.Image + """ + im = ensure_pil(im) + im.filter(ImageFilter.SHARPEN) + return ensure_np(im) + +def fit_image(im,targ_size): + """Force fit image by cropping + param: im: PIL.Image + param: targ_size: a tuple of target (width, height) + returns: PIL.Image + """ + im_pil = ensure_pil(im) + frame_pil = ImageOps.fit(im_pil, targ_size, + method=Image.BICUBIC, centering=(0.5, 0.5)) + return ensure_np(frame_pil) + + +def compute_entropy(im): + entr_img = entropy(im, disk(10)) + + +############################################ +# scikit-learn +############################################ + +def compute_entropy(im): + # im is grayscale numpy + return entropy(im, disk(10)) + +############################################ +# OpenCV +############################################ + +def bgr2gray(im): + """Wrapper for cv2.cvtColor transform + :param im: Numpy.ndarray (BGR) + :returns: Numpy.ndarray (Gray) + """ + return cv.cvtColor(im,cv.COLOR_BGR2GRAY) + +def gray2bgr(im): + """Wrapper for cv2.cvtColor transform + :param im: Numpy.ndarray (Gray) + :returns: Numpy.ndarray (BGR) + """ + return cv.cvtColor(im,cv.COLOR_GRAY2BGR) + +def bgr2rgb(im): + """Wrapper for cv2.cvtColor transform + :param im: Numpy.ndarray (BGR) + :returns: Numpy.ndarray (RGB) + """ + return cv.cvtColor(im,cv.COLOR_BGR2RGB) + +def compute_laplacian(im): + # below 100 is usually blurry + return cv.Laplacian(im, cv.CV_64F).var() + + +# http://radjkarl.github.io/imgProcessor/index.html# + +def modifiedLaplacian(img): + ''''LAPM' algorithm (Nayar89)''' + M = np.array([-1, 2, -1]) + G = cv.getGaussianKernel(ksize=3, sigma=-1) + Lx = cv.sepFilter2D(src=img, ddepth=cv.CV_64F, kernelX=M, kernelY=G) + Ly = cv.sepFilter2D(src=img, ddepth=cv.CV_64F, kernelX=G, kernelY=M) + FM = np.abs(Lx) + np.abs(Ly) + return cv.mean(FM)[0] + +def varianceOfLaplacian(img): + ''''LAPV' algorithm (Pech2000)''' + lap = cv.Laplacian(img, ddepth=-1)#cv.cv.CV_64F) + stdev = cv.meanStdDev(lap)[1] + s = stdev[0]**2 + return s[0] + +def tenengrad(img, ksize=3): + ''''TENG' algorithm (Krotkov86)''' + Gx = cv.Sobel(img, ddepth=cv.CV_64F, dx=1, dy=0, ksize=ksize) + Gy = cv.Sobel(img, ddepth=cv.CV_64F, dx=0, dy=1, ksize=ksize) + FM = Gx**2 + Gy**2 + return cv.mean(FM)[0] + +def normalizedGraylevelVariance(img): + ''''GLVN' algorithm (Santos97)''' + mean, stdev = cv.meanStdDev(img) + s = stdev[0]**2 / mean[0] + return s[0] + +def compute_if_blank(im,width=100,sigma=0,thresh_canny=.1,thresh_mean=4,mask=None): + # im is graysacale np + #im = imutils.resize(im,width=width) + #mask = imutils.resize(mask,width=width) + if mask is not None: + im_canny = feature.canny(im,sigma=sigma,mask=mask) + total = len(np.where(mask > 0)[0]) + else: + im_canny = feature.canny(im,sigma=sigma) + total = (im.shape[0]*im.shape[1]) + n_white = len(np.where(im_canny > 0)[0]) + per = n_white/total + if np.mean(im) < thresh_mean or per < thresh_canny: + return 1 + else: + return 0 + + +def print_timing(t,n): + t = time.time()-t + print('Elapsed time: {:.2f}'.format(t)) + print('FPS: {:.2f}'.format(n/t)) + +def vid2frames(fpath, limit=5000, width=None, idxs=None): + """Convert a video file into list of frames + :param fpath: filepath to the video file + :param limit: maximum number of frames to read + :param fpath: the indices of frames to keep (rest are skipped) + :returns: (fps, number of frames, list of Numpy.ndarray frames) + """ + frames = [] + try: + cap = cv.VideoCapture(fpath) + except: + print('[-] Error. Could not read video file: {}'.format(fpath)) + try: + cap.release() + except: + pass + return frames + + fps = cap.get(cv.CAP_PROP_FPS) + nframes = int(cap.get(cv.CAP_PROP_FRAME_COUNT)) + + if idxs is not None: + # read sample indices by seeking to frame index + for idx in idxs: + cap.set(cv.CAP_PROP_POS_FRAMES, idx) + res, frame = cap.read() + if width is not None: + frame = imutils.resize(frame, width=width) + frames.append(frame) + else: + while(True and len(frames) < limit): + res, frame = cap.read() + if not res: + break + if width is not None: + frame = imutils.resize(frame, width=width) + frames.append(frame) + + cap.release() + del cap + #return fps,nframes,frames + return frames + +def convolve_filter(vals,filters=[1]): + for k in filters: + vals_tmp = np.zeros_like(vals) + t = len(vals_tmp) + for i,v in enumerate(vals): + sum_vals = vals[max(0,i-k):min(t-1,i+k)] + vals_tmp[i] = np.mean(sum_vals) + vals = vals_tmp.copy() + return vals + +def cosine_delta(v1,v2): + return 1.0 - cosine_similarity(v1.reshape((1, -1)), v2.reshape((1, -1)))[0][0] + + + +def compute_edges(vals): + # find edges (1 = rising, -1 = falling) + edges = np.zeros_like(vals) + for i in range(len(vals[1:])): + delta = vals[i] - vals[i-1] + if delta == -1: + edges[i] = 1 # rising edge 0 --> 1 + elif delta == 1: + edges[i+1] = 2 # falling edge 1 --> 0 + # get index for rise fall + rising = np.where(np.array(edges) == 1)[0] + falling = np.where(np.array(edges) == 2)[0] + return rising, falling + + +############################################ +# Point, Rect +############################################ + +class Point(object): + def __init__(self, x, y): + self.x = x + self.y = y + +class Rect(object): + def __init__(self, p1, p2): + '''Store the top, bottom, left and right values for points + p1 and p2 are the (corners) in either order + ''' + self.left = min(p1.x, p2.x) + self.right = max(p1.x, p2.x) + self.top = min(p1.y, p2.y) + self.bottom = max(p1.y, p2.y) + +def overlap(r1, r2): + '''Overlapping rectangles overlap both horizontally & vertically + ''' + return range_overlap(r1.left, r1.right, r2.left, r2.right) and \ + range_overlap(r1.top, r1.bottom, r2.top, r2.bottom) + +def range_overlap(a_min, a_max, b_min, b_max): + '''Neither range is completely greater than the other + ''' + return (a_min <= b_max) and (b_min <= a_max) + +def merge_rects(r1,r2): + p1 = Point(min(r1.left,r2.left),min(r1.top,r2.top)) + p2 = Point(max(r1.right,r2.right),max(r1.bottom,r2.bottom)) + return Rect(p1,p2) + +def is_overlapping(r1,r2): + """r1,r2 as [x1,y1,x2,y2] list""" + r1x = Rect(Point(r1[0],r1[1]),Point(r1[2],r1[3])) + r2x = Rect(Point(r2[0],r2[1]),Point(r2[2],r2[3])) + return overlap(r1x,r2x) + +def get_rects_merged(rects,bounds,expand=0): + """rects: list of points in [x1,y1,x2,y2] format""" + rects_expanded = [] + bx,by = bounds + # expand + for x1,y1,x2,y2 in rects: + x1 = max(0,x1-expand) + y1 = max(0,y1-expand) + x2 = min(bx,x2+expand) + y2 = min(by,y2+expand) + rects_expanded.append(Rect(Point(x1,y1),Point(x2,y2))) + + #rects_expanded = [Rect(Point(x1,y1),Point(x2,y2)) for x1,y1,x2,y2 in rects_expanded] + rects_merged = [] + for i,r in enumerate(rects_expanded): + found = False + for j,rm in enumerate(rects_merged): + if overlap(r,rm): + rects_merged[j] = merge_rects(r,rm) #expand + found = True + if not found: + rects_merged.append(r) + # convert back to [x1,y1,x2,y2] format + rects_merged = [(r.left,r.top,r.right,r.bottom) for r in rects_merged] + # contract + rects_contracted = [] + for x1,y1,x2,y2 in rects_merged: + x1 = min(bx,x1+expand) + y1 = min(by,y1+expand) + x2 = max(0,x2-expand) + y2 = max(0,y2-expand) + rects_contracted.append((x1,y1,x2,y2)) + + return rects_contracted + + +############################################ +# Image display +############################################ + + +def montage(frames,ncols=4,nrows=None,width=None): + """Convert list of frames into a grid montage + param: frames: list of frames as Numpy.ndarray + param: ncols: number of columns + param: width: resize images to this width before adding to grid + returns: Numpy.ndarray grid of all images + """ + + # expand image size if not enough frames + if nrows is not None and len(frames) < ncols * nrows: + blank = np.zeros_like(frames[0]) + n = ncols * nrows - len(frames) + for i in range(n): frames.append(blank) + + rows = [] + for i,im in enumerate(frames): + if width is not None: + im = imutils.resize(im,width=width) + h,w = im.shape[:2] + if i % ncols == 0: + if i > 0: + rows.append(ims) + ims = [] + ims.append(im) + if len(ims) > 0: + for j in range(ncols-len(ims)): + ims.append(np.zeros_like(im)) + rows.append(ims) + row_ims = [] + for row in rows: + row_im = np.hstack(np.array(row)) + row_ims.append(row_im) + contact_sheet = np.vstack(np.array(row_ims)) + return contact_sheet diff --git a/megapixels/app/utils/logger_utils.py b/megapixels/app/utils/logger_utils.py new file mode 100644 index 00000000..d4f962eb --- /dev/null +++ b/megapixels/app/utils/logger_utils.py @@ -0,0 +1,68 @@ +""" +Logger instantiator for use with Click utlity scripts +""" +import sys +import os +import logging + +import colorlog + +from app.settings import app_cfg as cfg + + +class Logger: + + logger_name = 'app' + + def __init__(self): + pass + + @staticmethod + def create(verbosity=4, logfile=None): + """Configures a logger from click params + :param verbosity: (int) between 0 and 5 + :param logfile: (str) path to logfile + :returns: logging root object + """ + + loglevel = (5 - (max(0, min(verbosity, 5)))) * 10 # where logging.DEBUG = 10 + date_format = '%Y-%m-%d %H:%M:%S' + if 'colorlog' in sys.modules and os.isatty(2): + cformat = '%(log_color)s' + cfg.LOGFILE_FORMAT + f = colorlog.ColoredFormatter(cformat, date_format, + log_colors = { 'DEBUG' : 'yellow', 'INFO' : 'white', + 'WARNING' : 'bold_yellow', 'ERROR': 'bold_red', + 'CRITICAL': 'bold_red' }) + else: + f = logging.Formatter(cfg.LOGFILE_FORMAT, date_format) + + # logger = logging.getLogger(Logger.logger_name) + logger = logging.getLogger(cfg.LOGGER_NAME) + logger.setLevel(loglevel) + + if logfile: + # create file handler which logs even debug messages + fh = logging.FileHandler(logfile) + fh.setLevel(loglevel) + logger.addHandler(fh) + + # add colored handler + ch = logging.StreamHandler() + ch.setFormatter(f) + logger.addHandler(ch) + + if verbosity == 0: + logger.disabled = True + + # test + # logger.debug('Hello Debug') + # logger.info('Hello Info') + # logger.warn('Hello Warn') + # logger.error('Hello Error') + # logger.critical('Hello Critical') + + return logger + + @staticmethod + def getLogger(): + return logging.getLogger(cfg.LOGGER_NAME) \ No newline at end of file diff --git a/megapixels/cli_admin.py b/megapixels/cli_admin.py new file mode 100644 index 00000000..45ebeed4 --- /dev/null +++ b/megapixels/cli_admin.py @@ -0,0 +1,36 @@ +# -------------------------------------------------------- +# This is the vframe administrative script for utility +# add/edit commands in vframe/admin/commands directory +# -------------------------------------------------------- +import click + +from app.settings import app_cfg as cfg +from app.utils import logger_utils +from app.models.click_factory import ClickSimple + +# click cli factory +cc = ClickSimple.create(cfg.DIR_COMMANDS_PROCESSOR_ADMIN) + +# -------------------------------------------------------- +# CLI +# -------------------------------------------------------- +@click.group(cls=cc, chain=False) +@click.option('-v', '--verbose', 'verbosity', count=True, default=4, + show_default=True, + help='Verbosity: -v DEBUG, -vv INFO, -vvv WARN, -vvvv ERROR, -vvvvv CRITICAL') +@click.pass_context +def cli(ctx, **kwargs): + """\033[1m\033[94mMegaPixels: Admin/Utility Scripts\033[0m + """ + ctx.opts = {} + # init logger + logger_utils.Logger.create(verbosity=kwargs['verbosity']) + + + +# -------------------------------------------------------- +# Entrypoint +# -------------------------------------------------------- +if __name__ == '__main__': + cli() + diff --git a/megapixels/cli_datasets.py b/megapixels/cli_datasets.py new file mode 100644 index 00000000..ae484e80 --- /dev/null +++ b/megapixels/cli_datasets.py @@ -0,0 +1,36 @@ +# -------------------------------------------------------- +# This is the vframe administrative script for utility +# add/edit commands in vframe/admin/commands directory +# -------------------------------------------------------- +import click + +from app.settings import app_cfg as cfg +from app.utils import logger_utils +from app.models.click_factory import ClickSimple + +# click cli factory +cc = ClickSimple.create(cfg.DIR_COMMANDS_PROCESSOR_DATASETS) + +# -------------------------------------------------------- +# CLI +# -------------------------------------------------------- +@click.group(cls=cc, chain=False) +@click.option('-v', '--verbose', 'verbosity', count=True, default=4, + show_default=True, + help='Verbosity: -v DEBUG, -vv INFO, -vvv WARN, -vvvv ERROR, -vvvvv CRITICAL') +@click.pass_context +def cli(ctx, **kwargs): + """\033[1m\033[94mMegaPixels: Admin/Utility Scripts\033[0m + """ + ctx.opts = {} + # init logger + logger_utils.Logger.create(verbosity=kwargs['verbosity']) + + + +# -------------------------------------------------------- +# Entrypoint +# -------------------------------------------------------- +if __name__ == '__main__': + cli() + diff --git a/megapixels/datasets/commands/crop.py b/megapixels/datasets/commands/crop.py new file mode 100644 index 00000000..778be0c4 --- /dev/null +++ b/megapixels/datasets/commands/crop.py @@ -0,0 +1,104 @@ +""" +Crop images to prepare for training +""" + +import click +from PIL import Image, ImageOps, ImageFilter, ImageDraw + +from app.settings import types +from app.utils import click_utils +from app.settings import app_cfg as cfg + +@click.command() +@click.option('-i', '--input', 'opt_dir_in', required=True, + help='Input directory') +@click.option('-o', '--output', 'opt_dir_out', required=True, + help='Output directory') +@click.option('-e', '--ext', 'opt_ext', + default='jpg', type=click.Choice(['jpg', 'png']), + help='File glob ext') +@click.option('--size', 'opt_size', + type=(int, int), default=(256, 256), + help='Output image size') +@click.option('-t', '--crop-type', 'opt_crop_type', + default='center', type=click.Choice(['center', 'mirror', 'face', 'person', 'none']), + help='Force fit image center location') +@click.pass_context +def cli(ctx, opt_dir_in, opt_dir_out, opt_ext, opt_size, opt_crop_type): + """Crop, mirror images""" + + import os + from os.path import join + from pathlib import Path + from glob import glob + from tqdm import tqdm + + + from app.utils import logger_utils, file_utils, im_utils + + # ------------------------------------------------- + # process here + + log = logger_utils.Logger.getLogger() + log.info('crop images') + + # get list of files to process + fp_ims = glob(join(opt_dir_in, '*.{}'.format(opt_ext))) + log.debug('files: {}'.format(len(fp_ims))) + + # ensure output dir exists + file_utils.mkdirs(opt_dir_out) + + for fp_im in tqdm(fp_ims): + im = process_crop(fp_im, opt_size, opt_crop_type) + fp_out = join(opt_dir_out, Path(fp_im).name) + im.save(fp_out) + + +def process_crop(fp_im, opt_size, crop_type): + im = Image.open(fp_im) + if crop_type == 'center': + im = crop_square_fit(im, opt_size) + elif crop_type == 'mirror': + im = mirror_crop_square(im, opt_size) + return im + +def crop_square_fit(im, size, center=(0.5, 0.5)): + return ImageOps.fit(im, size, method=Image.BICUBIC, centering=center) + +def mirror_crop_square(im, size): + # force to even dims + if im.size[0] % 2 or im.size[1] % 2: + im = ImageOps.fit(im, ((im.size[0] // 2) * 2, (im.size[1] // 2) * 2)) + + # create new square image + min_size, max_size = (min(im.size), max(im.size)) + orig_w, orig_h = im.size + margin = (max_size - min_size) // 2 + w, h = (max_size, max_size) + im_new = Image.new('RGB', (w, h), color=(0, 0, 0)) + + #crop (l, t, r, b) + if orig_w > orig_h: + # landscape, mirror expand T/B + im_top = ImageOps.mirror(im.crop((0, 0, margin, w))) + im_bot = ImageOps.mirror(im.crop((orig_h - margin, 0, orig_h, w))) + im_new.paste(im_top, (0, 0)) + im_new.paste(im, (margin, 0, orig_h + margin, w)) + im_new.paste(im_bot, (h - margin, 0)) + elif orig_h > orig_w: + # portrait, mirror expand L/R + im_left = ImageOps.mirror(im.crop((0, 0, margin, h))) + im_right = ImageOps.mirror(im.crop((orig_w - margin, 0, orig_w, h))) + im_new.paste(im_left, (0, 0)) + im_new.paste(im, (margin, 0, orig_w + margin, h)) + im_new.paste(im_right, (w - margin, 0)) + + return im_new.resize(size) + + +def center_crop_face(): + pass + +def center_crop_person(): + pass \ No newline at end of file diff --git a/megapixels/datasets/commands/extract.py b/megapixels/datasets/commands/extract.py new file mode 100644 index 00000000..4e77a978 --- /dev/null +++ b/megapixels/datasets/commands/extract.py @@ -0,0 +1,86 @@ +""" +Crop images to prepare for training +""" + +import click + +from app.settings import types +from app.utils import click_utils +from app.settings import app_cfg as cfg + +@click.command() +@click.option('-i', '--input', 'opt_fp_in', required=True, + help='Input CSV') +@click.option('--media', 'opt_dir_media', required=True, + help='Input image/video directory') +@click.option('-o', '--output', 'opt_dir_out', required=True, + help='Output directory for extracted ROI images') +@click.option('--size', 'opt_size', + type=(int, int), default=(300, 300), + help='Output image size') +@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None), + help='Slice list of files') +@click.option('--padding', 'opt_padding', default=0, + help='Facial padding') +@click.option('--ext', 'opt_ext_out', default='jpg', type=click.Choice(['jpg', 'png']), + help='Output image type') +@click.pass_context +def cli(ctx, opt_fp_in, opt_dir_media, opt_dir_out, opt_size, opt_slice, + opt_padding, opt_ext_out): + """Extrace ROIs to images""" + + import os + from os.path import join + from pathlib import Path + from glob import glob + + from tqdm import tqdm + import numpy as np + from PIL import Image, ImageOps, ImageFilter, ImageDraw + import cv2 as cv + import pandas as pd + + from app.utils import logger_utils, file_utils, im_utils + from app.models.bbox import BBox + # ------------------------------------------------- + # process here + log = logger_utils.Logger.getLogger() + + df_rois = pd.read_csv(opt_fp_in) + if opt_slice: + df_rois = df_rois[opt_slice[0]:opt_slice[1]] + + log.info('Processing {:,} rows'.format(len(df_rois))) + + file_utils.mkdirs(opt_dir_out) + + df_rois_grouped = df_rois.groupby(['fn']) # group by fn/filename + groups = df_rois_grouped.groups + + for group in groups: + + # get image + group_rows = df_rois_grouped.get_group(group) + + row = group_rows.iloc[0] + fp_im = join(opt_dir_media, '{fn}{ext}'.format(**row)) #TODO change to ext + im = Image.open(fp_im) + + + for idx, roi in group_rows.iterrows(): + log.info('{}'.format(roi['fn'])) + # get bbox to im dimensions + xywh = [roi['x'], roi['y'], roi['w'] , roi['h']] + bbox = BBox.from_xywh(*xywh) + dim = im.size + bbox_dim = bbox.to_dim(dim) + # expand + bbox_dim_exp = bbox_dim.expand_dim(opt_padding, dim) + # crop + x1y2 = bbox_dim_exp.pt_tl + bbox_dim_exp.pt_br + im_crop = im.crop(box=x1y2) + # save + idx_zpad = file_utils.zpad(idx, zeros=3) + fp_im_out = join(opt_dir_out, '{}_{}.{}'.format(roi['fn'], idx_zpad, opt_ext_out)) + im_crop.save(fp_im_out) + diff --git a/megapixels/datasets/commands/face.py b/megapixels/datasets/commands/face.py new file mode 100644 index 00000000..6b7b18b7 --- /dev/null +++ b/megapixels/datasets/commands/face.py @@ -0,0 +1,117 @@ +""" +Crop images to prepare for training +""" + +import click +# from PIL import Image, ImageOps, ImageFilter, ImageDraw + +from app.settings import types +from app.utils import click_utils +from app.settings import app_cfg as cfg + +@click.command() +@click.option('-i', '--input', 'opt_dir_in', required=True, + help='Input directory') +@click.option('-o', '--output', 'opt_fp_out', required=True, + help='Output CSV') +@click.option('-e', '--ext', 'opt_ext', + default='jpg', type=click.Choice(['jpg', 'png']), + help='File glob ext') +@click.option('--size', 'opt_size', + type=(int, int), default=(300, 300), + help='Output image size') +@click.option('-t', '--detector-type', 'opt_detector_type', + type=cfg.FaceDetectNetVar, + default=click_utils.get_default(types.FaceDetectNet.DLIB_CNN), + help=click_utils.show_help(types.FaceDetectNet)) +@click.option('-g', '--gpu', 'opt_gpu', default=0, + help='GPU index') +@click.option('--conf', 'opt_conf_thresh', default=0.85, type=click.FloatRange(0,1), + help='Confidence minimum threshold') +@click.option('--pyramids', 'opt_pyramids', default=0, type=click.IntRange(0,4), + help='Number pyramids to upscale for DLIB detectors') +@click.option('--slice', 'opt_slice', type=(int, int), default=(None, None), + help='Slice list of files') +@click.option('--display/--no-display', 'opt_display', is_flag=True, default=False, + help='Display detections to debug') +@click.pass_context +def cli(ctx, opt_dir_in, opt_fp_out, opt_ext, opt_size, opt_detector_type, + opt_gpu, opt_conf_thresh, opt_pyramids, opt_slice, opt_display): + """Extrace face""" + + import sys + import os + from os.path import join + from pathlib import Path + from glob import glob + from tqdm import tqdm + import numpy as np + import dlib # must keep a local reference for dlib + import cv2 as cv + import pandas as pd + + from app.utils import logger_utils, file_utils, im_utils + from app.processors import face_detector + + # ------------------------------------------------- + # init here + + log = logger_utils.Logger.getLogger() + + if opt_detector_type == types.FaceDetectNet.CVDNN: + detector = face_detector.DetectorCVDNN() + elif opt_detector_type == types.FaceDetectNet.DLIB_CNN: + detector = face_detector.DetectorDLIBCNN(opt_gpu) + elif opt_detector_type == types.FaceDetectNet.DLIB_HOG: + detector = face_detector.DetectorDLIBHOG() + elif opt_detector_type == types.FaceDetectNet.HAAR: + log.error('{} not yet implemented'.format(opt_detector_type.name)) + return + + + # ------------------------------------------------- + # process here + + # get list of files to process + fp_ims = glob(join(opt_dir_in, '*.{}'.format(opt_ext))) + if opt_slice: + fp_ims = fp_ims[opt_slice[0]:opt_slice[1]] + log.debug('processing {:,} files'.format(len(fp_ims))) + + + data = [] + + for fp_im in tqdm(fp_ims): + im = cv.imread(fp_im) + bboxes = detector.detect(im, opt_size=opt_size, opt_pyramids=opt_pyramids) + fpp_im = Path(fp_im) + for bbox in bboxes: + roi = { + 'fn': fpp_im.stem, + 'ext': fpp_im.suffix, + 'x': bbox.x, + 'y': bbox.y, + 'w': bbox.w, + 'h': bbox.h} + dim = bbox.to_dim(im.shape[:2][::-1]) # w,h + data.append(roi) + + # debug display + if opt_display and len(bboxes): + im_md = im_utils.resize(im, width=opt_size[0]) + for bbox in bboxes: + dim = bbox.to_dim(im_md.shape[:2][::-1]) + cv.rectangle(im_md, dim.pt_tl, dim.pt_br, (0,255,0), 3) + cv.imshow('', im_md) + while True: + k = cv.waitKey(1) & 0xFF + if k == 27 or k == ord('q'): # ESC + cv.destroyAllWindows() + sys.exit() + elif k != 255: + # any key to continue + break + + # save date + df = pd.DataFrame.from_dict(data) + df.to_csv(opt_fp_out) \ No newline at end of file diff --git a/megapixels/datasets/commands/resize.py b/megapixels/datasets/commands/resize.py new file mode 100644 index 00000000..5e2d31aa --- /dev/null +++ b/megapixels/datasets/commands/resize.py @@ -0,0 +1,81 @@ +""" +Crop images to prepare for training +""" + +import click + +from app.settings import types +from app.utils import click_utils +from app.settings import app_cfg as cfg + +""" +Filter Q-Down Q-Up Speed +NEAREST ⭐⭐⭐⭐⭐ +BOX ⭐ ⭐⭐⭐⭐ +BILINEAR ⭐ ⭐ ⭐⭐⭐ +HAMMING ⭐⭐ ⭐⭐⭐ +BICUBIC ⭐⭐⭐ ⭐⭐⭐ ⭐⭐ +LANCZOS ⭐⭐⭐⭐ ⭐⭐⭐⭐ ⭐ +""" + +@click.command() +@click.option('-i', '--input', 'opt_dir_in', required=True, + help='Input directory') +@click.option('-o', '--output', 'opt_dir_out', required=True, + help='Output directory') +@click.option('-e', '--ext', 'opt_glob_ext', + default='jpg', type=click.Choice(['jpg', 'png']), + help='File glob ext') +@click.option('--size', 'opt_size', + type=(int, int), default=(256, 256), + help='Output image size (square)') +@click.option('--method', 'opt_scale_method', + type=click.Choice(['LANCZOS', 'BICUBIC', 'HAMMING', 'BILINEAR', 'BOX', 'NEAREST']), + default='LANCZOS', + help='Scaling method to use') +@click.pass_context +def cli(ctx, opt_dir_in, opt_dir_out, opt_glob_ext, opt_size, opt_scale_method): + """Crop, mirror images""" + + import os + from os.path import join + from pathlib import Path + from glob import glob + from tqdm import tqdm + from PIL import Image, ImageOps, ImageFilter + from app.utils import logger_utils, file_utils, im_utils + + # ------------------------------------------------- + # init + + log = logger_utils.Logger.getLogger() + + methods = { + 'LANCZOS': Image.LANCZOS, + 'BICUBIC': Image.BICUBIC, + 'HAMMING': Image.HAMMING, + 'BILINEAR': Image.BILINEAR, + 'BOX': Image.BOX, + 'NEAREST': Image.NEAREST + } + + # ------------------------------------------------- + # process here + + # get list of files to process + fp_ims = glob(join(opt_dir_in, '*.{}'.format(opt_glob_ext))) + log.info('processing {:,} files'.format(len(fp_ims))) + + # set scale method + scale_method = methods[opt_scale_method] + + # ensure output dir exists + file_utils.mkdirs(opt_dir_out) + + # resize and save images + for fp_im in tqdm(fp_ims): + im = Image.open(fp_im) + im = ImageOps.fit(im, opt_size, method=scale_method, centering=(0.5, 0.5)) + fp_out = join(opt_dir_out, Path(fp_im).name) + im.save(fp_out) + diff --git a/notes.md b/notes.md new file mode 100644 index 00000000..9dcf3da6 --- /dev/null +++ b/notes.md @@ -0,0 +1,70 @@ +PATH=/home/adam/torch/install/bin:/home/adam/anaconda3/bin:/home/adam/anaconda3/envs/megapixels/bin:/home/adam/anaconda3/bin:/home/adam/.nvm/versions/node/v9.9.0/bin:/home/adam/bin:/home/adam/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/usr/lib/nvidia/:/usr/local/cuda/bin:/usr/lib/nvidia/:/usr/local/cuda/bin + +PATH=/home/adam/anaconda3/bin:/home/adam/.nvm/versions/node/v9.9.0/bin:/home/adam/code/google-cloud-sdk/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/usr/local/cuda/bin + +CUDA_HOME=/usr/local/cuda +LD_LIBRARY_PATH=/home/adam/torch/install/lib::/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64 + + + +LD_LIBRARY_PATH=/home/adam/torch/install/lib::/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64 + + +LD_LIBRARY_PATH=:/usr/local/cuda/lib64 +CUDA_HOME=/usr/local/cuda + + +TORCH_NVCC_FLAGS=-D__CUDA_NO_HALF_OPERATORS__ +TORCH_NVCC_FLAGS=-D__CUDA_NO_HALF_OPERATORS__ + + +export PATH=/usr/local/cuda/bin:"$PATH +./clean.sh +export TORCH_NVCC_FLAGS="-D__CUDA_NO_HALF_OPERATORS__" +./install.sh + +find . -name "*.JPEG" | xargs -I {} convert {} -resize "256^>" {} + +find . -name \*.png -exec identify -ping {} \; -or -exec echo {} \; +find . -name \*.jpg -exec identify -ping {} \; -or -exec rm -f {} \; + +luarocks install cudnn +luarocks install display + +scp undisclosed:/home/adam/FIDs.zip . +unzip -q FIDs.zip +find FIDs_NEW -name \*.jpg > list.txt +mkdir -p /work/megapixels_dev/3rdparty/art-DCGAN/fiw/images/ + +while read -r line;do dst=/work/megapixels_dev/3rdparty/art-DCGAN/hipsterwars/images/$(basename "$line");src=`pwd`/$line;ln -s $src $dst;done < list.txt + +extension="${filename##*.}" + + +filename="${filename%.*} + +for d in $(find source -type d) + do + ls $d/*.bin 1>/dev/null 2>&1 && ln -s $d/*.bin target/$(basename $d).dat;done + +gpu=0 batchSize=1 imsize=10 noisemode=linefull net=bedrooms_4_net_G.t7 th generate.lua + +DATA_ROOT=fiw dataset=folder ndf=50 ngf=150 name=fiw_01 nThreads=6 gpu=2 th main.lua + +DATA_ROOT=megaface_13 dataset=folder ndf=50 ngf=150 name=megaface_13 nThreads=6 gpu=1 th main.lua + +DATA_ROOT=hipsterwars dataset=folder ndf=50 ngf=150 name=hipsterwars nThreads=6 gpu=2 th main.lua + +export PATH=/usr/local/cuda/bin/:$PATH + +export LD_LIBRARY_PATH=/usr/local/cuda/lib64/:$LD_LIBRARY_PATH + + +git clone https://github.com/soumith/cudnn.torch.git -b R7 && cd cudnn.torch && luarocks make cudnn-scm-1.rockspec + +scp undisclosed:/home/adam/hipsterwars_v1.0.zip . +find . -name "*.jpg" -print0 | xargs -0 mogrify -flop + +https://github.com/facebookresearch/deepcluster + +DATA_ROOT=hipsterwars dataset=folder ndf=100 ngf=200 batchSize=128 name=hipsterwars_d100_g200_b128 nThreads=8 gpu=2 th main.lua && DATA_ROOT=hipsterwars dataset=folder ndf=100 ngf=200 batchSize=64 name=hipsterwars_d100_g200_b64 nThreads=8 gpu=2 th main.lua && DATA_ROOT=hipsterwars dataset=folder ndf=100 ngf=300 batchSize=128 name=hipsterwars_d100_g300_b128 nThreads=8 gpu=2 th main.lua && DATA_ROOT=hipsterwars dataset=folder ndf=100 ngf=300 batchSize=64 name=hipsterwars_d100_g200_b64 nThreads=8 gpu=2 th main.lua \ No newline at end of file diff --git a/notes/cheat-sheets/bash-cheat-sheet.sh b/notes/cheat-sheets/bash-cheat-sheet.sh new file mode 100644 index 00000000..dee67440 --- /dev/null +++ b/notes/cheat-sheets/bash-cheat-sheet.sh @@ -0,0 +1,410 @@ +# bash cheat sheet + +#!/bin/bash +##################################################### +# Name: Bash CheatSheet for Mac OSX +# +# A little overlook of the Bash basics +# +# Usage: +# +# Author: J. Le Coupanec +# Date: 2014/11/04 +##################################################### + + +# 0. Shortcuts. + + +CTRL+A # move to beginning of line +CTRL+B # moves backward one character +CTRL+C # halts the current command +CTRL+D # deletes one character backward or logs out of current session, similar to exit +CTRL+E # moves to end of line +CTRL+F # moves forward one character +CTRL+G # aborts the current editing command and ring the terminal bell +CTRL+J # same as RETURN +CTRL+K # deletes (kill) forward to end of line +CTRL+L # clears screen and redisplay the line +CTRL+M # same as RETURN +CTRL+N # next line in command history +CTRL+O # same as RETURN, then displays next line in history file +CTRL+P # previous line in command history +CTRL+R # searches backward +CTRL+S # searches forward +CTRL+T # transposes two characters +CTRL+U # kills backward from point to the beginning of line +CTRL+V # makes the next character typed verbatim +CTRL+W # kills the word behind the cursor +CTRL+X # lists the possible filename completefions of the current word +CTRL+Y # retrieves (yank) last item killed +CTRL+Z # stops the current command, resume with fg in the foreground or bg in the background + +DELETE # deletes one character backward +!! # repeats the last command +exit # logs out of current session + + +# 1. Bash Basics. + + +export # displays all environment variables + +echo $SHELL # displays the shell you're using +echo $BASH_VERSION # displays bash version + +bash # if you want to use bash (type exit to go back to your normal shell) +whereis bash # finds out where bash is on your system + +clear # clears content on window (hide displayed lines) + + +# 1.1. File Commands. + + +ls # lists your files +ls -l # lists your files in 'long format', which contains the exact size of the file, who owns the file and who has the right to look at it, and when it was last modified +ls -a # lists all files, including hidden files +ln -s # creates symbolic link to file +touch # creates or updates your file +cat > # places standard input into file +more # shows the first part of a file (move with space and type q to quit) +head # outputs the first 10 lines of file +tail # outputs the last 10 lines of file (useful with -f option) +emacs # lets you create and edit a file +mv # moves a file +cp # copies a file +rm # removes a file +diff # compares files, and shows where they differ +wc # tells you how many lines, words and characters there are in a file +chmod -options # lets you change the read, write, and execute permissions on your files +gzip # compresses files +gunzip # uncompresses files compressed by gzip +gzcat # lets you look at gzipped file without actually having to gunzip it +lpr # print the file +lpq # check out the printer queue +lprm # remove something from the printer queue +genscript # converts plain text files into postscript for printing and gives you some options for formatting +dvips # print .dvi files (i.e. files produced by LaTeX) +grep # looks for the string in the files +grep -r # search recursively for pattern in directory + + +# 1.2. Directory Commands. + + +mkdir # makes a new directory +cd # changes to home +cd # changes directory +pwd # tells you where you currently are + + +# 1.3. SSH, System Info & Network Commands. + + +ssh user@host # connects to host as user +ssh -p user@host # connects to host on specified port as user +ssh-copy-id user@host # adds your ssh key to host for user to enable a keyed or passwordless login + +whoami # returns your username +passwd # lets you change your password +quota -v # shows what your disk quota is +date # shows the current date and time +cal # shows the month's calendar +uptime # shows current uptime +w # displays whois online +finger # displays information about user +uname -a # shows kernel information +man # shows the manual for specified command +df # shows disk usage +du # shows the disk usage of the files and directories in filename (du -s give only a total) +last # lists your last logins +ps -u yourusername # lists your processes +kill # kills (ends) the processes with the ID you gave +killall # kill all processes with the name +top # displays your currently active processes +bg # lists stopped or background jobs ; resume a stopped job in the background +fg # brings the most recent job in the foreground +fg # brings job to the foreground + +ping # pings host and outputs results +whois # gets whois information for domain +dig # gets DNS information for domain +dig -x # reverses lookup host +wget # downloads file + + +# 2. Basic Shell Programming. + + +# 2.1. Variables. + + +varname=value # defines a variable +varname=value command # defines a variable to be in the environment of a particular subprocess +echo $varname # checks a variable's value +echo $$ # prints process ID of the current shell +echo $! # prints process ID of the most recently invoked background job +echo $? # displays the exit status of the last command +export VARNAME=value # defines an environment variable (will be available in subprocesses) + +array[0] = val # several ways to define an array +array[1] = val +array[2] = val +array=([2]=val [0]=val [1]=val) +array(val val val) + +${array[i]} # displays array's value for this index. If no index is supplied, array element 0 is assumed +${#array[i]} # to find out the length of any element in the array +${#array[@]} # to find out how many values there are in the array + +declare -a # the variables are treaded as arrays +declare -f # uses funtion names only +declare -F # displays function names without definitions +declare -i # the variables are treaded as integers +declare -r # makes the variables read-only +declare -x # marks the variables for export via the environment + +${varname:-word} # if varname exists and isn't null, return its value; otherwise return word +${varname:=word} # if varname exists and isn't null, return its value; otherwise set it word and then return its value +${varname:?message} # if varname exists and isn't null, return its value; otherwise print varname, followed by message and abort the current command or script +${varname:+word} # if varname exists and isn't null, return word; otherwise return null +${varname:offset:length} # performs substring expansion. It returns the substring of $varname starting at offset and up to length characters + +${variable#pattern} # if the pattern matches the beginning of the variable's value, delete the shortest part that matches and return the rest +${variable##pattern} # if the pattern matches the beginning of the variable's value, delete the longest part that matches and return the rest +${variable%pattern} # if the pattern matches the end of the variable's value, delete the shortest part that matches and return the rest +${variable%%pattern} # if the pattern matches the end of the variable's value, delete the longest part that matches and return the rest +${variable/pattern/string} # the longest match to pattern in variable is replaced by string. Only the first match is replaced +${variable//pattern/string} # the longest match to pattern in variable is replaced by string. All matches are replaced + +${#varname} # returns the length of the value of the variable as a character string + +*(patternlist) # matches zero or more occurences of the given patterns ++(patternlist) # matches one or more occurences of the given patterns +?(patternlist) # matches zero or one occurence of the given patterns +@(patternlist) # matches exactly one of the given patterns +!(patternlist) # matches anything except one of the given patterns + +$(UNIX command) # command substitution: runs the command and returns standard output + + +# 2.2. Functions. +# The function refers to passed arguments by position (as if they were positional parameters), that is, $1, $2, and so forth. +# $@ is equal to "$1" "$2"... "$N", where N is the number of positional parameters. $# holds the number of positional parameters. + + +functname() { + shell commands +} + +unset -f functname # deletes a function definition +declare -f # displays all defined functions in your login session + + +# 2.3. Flow Control. + + +statement1 && statement2 # and operator +statement1 || statement2 # or operator + +-a # and operator inside a test conditional expression +-o # or operator inside a test conditional expression + +str1=str2 # str1 matches str2 +str1!=str2 # str1 does not match str2 +str1str2 # str1 is greater than str2 +-n str1 # str1 is not null (has length greater than 0) +-z str1 # str1 is null (has length 0) + +-a file # file exists +-d file # file exists and is a directory +-e file # file exists; same -a +-f file # file exists and is a regular file (i.e., not a directory or other special type of file) +-r file # you have read permission +-r file # file exists and is not empty +-w file # your have write permission +-x file # you have execute permission on file, or directory search permission if it is a directory +-N file # file was modified since it was last read +-O file # you own file +-G file # file's group ID matches yours (or one of yours, if you are in multiple groups) +file1 -nt file2 # file1 is newer than file2 +file1 -ot file2 # file1 is older than file2 + +-lt # less than +-le # less than or equal +-eq # equal +-ge # greater than or equal +-gt # greater than +-ne # not equal + +if condition +then + statements +[elif condition + then statements...] +[else + statements] +fi + +for x := 1 to 10 do +begin + statements +end + +for name [in list] +do + statements that can use $name +done + +for (( initialisation ; ending condition ; update )) +do + statements... +done + +case expression in + pattern1 ) + statements ;; + pattern2 ) + statements ;; + ... +esac + +select name [in list] +do + statements that can use $name +done + +while condition; do + statements +done + +until condition; do + statements +done + + +# 3. Command-Line Processing Cycle. + + +# The default order for command lookup is functions, followed by built-ins, with scripts and executables last. +# There are three built-ins that you can use to override this order: `command`, `builtin` and `enable`. + +command # removes alias and function lookup. Only built-ins and commands found in the search path are executed +builtin # looks up only built-in commands, ignoring functions and commands found in PATH +enable # enables and disables shell built-ins + +eval # takes arguments and run them through the command-line processing steps all over again + + +# 4. Input/Output Redirectors. + + +cmd1|cmd2 # pipe; takes standard output of cmd1 as standard input to cmd2 +> file # directs standard output to file +< file # takes standard input from file +>> file # directs standard output to file; append to file if it already exists +>|file # forces standard output to file even if noclobber is set +n>|file # forces output to file from file descriptor n even if noclobber is set +<> file # uses file as both standard input and standard output +n<>file # uses file as both input and output for file descriptor n +<