import os import sys sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../live-cortex/rpc/')) from options.test_options import TestOptions from options.dataset_options import DatasetOptions from data import CreateRecursiveDataLoader from models import create_model # from util.visualizer import Visualizer from util.util import mkdirs, crop_image from util import html from shutil import move, copyfile from PIL import Image, ImageOps from skimage.transform import resize from scipy.misc import imresize from shutil import copyfile, rmtree import numpy as np import cv2 from datetime import datetime import re import sys import math import subprocess from time import sleep from rpc import CortexRPC def clamp(n,a,b): return max(a, min(n, b)) def lerp(n,a,b): return (b-a)*n+a def load_opt(): opt_parser = TestOptions() opt = opt_parser.parse() data_opt_parser = DatasetOptions() data_opt = data_opt_parser.parse(opt.unknown) opt.nThreads = 1 # test code only supports nThreads = 1 opt.batchSize = 1 # test code only supports batchSize = 1 opt.serial_batches = True # no shuffle opt.no_flip = True # no flip data_opt.tag = get_tag(opt, data_opt) opt.render_dir = opt.results_dir + opt.name + "/" + data_opt.tag + "/" return opt, data_opt, data_opt_parser def get_tag(opt, data_opt): if data_opt.tag == '': d = datetime.now() tag = data_opt.tag = "{}_{}_{}".format( opt.name, 'live', d.strftime('%Y%m%d%H%M') ) else: tag = data_opt.tag return tag def load_first_frame(opt, data_opt): print("create render_dir: {}".format(opt.render_dir)) if os.path.exists(opt.render_dir): rmtree(opt.render_dir) mkdirs(opt.render_dir) start_img_path = os.path.join(opt.render_dir, "frame_00000.png") if data_opt.just_copy: copyfile(opt.start_img, start_img_path) A_im = None A_img = None A_offset = 0 else: print("preload {}".format(opt.start_img)) A_img = Image.open(opt.start_img).convert('RGB') A_im = np.asarray(A_img) A = process_image(opt, data_opt, A_im) cv2.imwrite(start_img_path, A) numz = re.findall(r'\d+', os.path.basename(opt.start_img)) print(numz) if len(numz) > 0: A_offset = int(numz[0]) print(A_offset) if A_offset: print(">> starting offset: {}".format(A_offset)) A_dir = opt.start_img.replace(numz[0], "{:05d}") print(A_dir) else: print("Sequence not found") return A_offset, A_im, A_dir def process_image(opt, data_opt, im): img = im[:, :, ::-1].copy() processed = False if data_opt.clahe is True: processed = True lab = cv2.cvtColor(img, cv2.COLOR_BGR2LAB) l, a, b = cv2.split(lab) clahe = cv2.createCLAHE(clipLimit=data_opt.clip_limit, tileGridSize=(8,8)) l = clahe.apply(l) limg = cv2.merge((l,a,b)) img = cv2.cvtColor(limg, cv2.COLOR_LAB2BGR) if data_opt.posterize is True: processed = True img = cv2.pyrMeanShiftFiltering(img, data_opt.spatial_window, data_opt.color_window) if data_opt.grayscale is True: processed = True img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) if data_opt.blur is True: processed = True img = cv2.GaussianBlur(img, (data_opt.blur_radius, data_opt.blur_radius), data_opt.blur_sigma) if data_opt.canny is True: processed = True img = cv2.Canny(img, data_opt.canny_lo, data_opt.canny_hi) img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR) if processed is False or data_opt.process_frac == 0: return img src_img = im[:, :, ::-1].copy() frac_a = data_opt.process_frac frac_b = 1.0 - frac_a array_a = np.multiply(src_img.astype('float64'), frac_a) array_b = np.multiply(img.astype('float64'), frac_b) img = np.add(array_a, array_b).astype('uint8') return img class Listener(): def __init__(self): opt, data_opt, data_opt_parser = load_opt() self.opt = opt self.data_opt = data_opt self.data_opt_parser = data_opt_parser.parser self.working = False def _set_fn(self, key, value): if hasattr(self.data_opt, key): try: if str(value) == 'True': setattr(self.data_opt, key, True) print('set {} {}: {}'.format('bool', key, True)) elif str(value) == 'False': setattr(self.data_opt, key, False) print('set {} {}: {}'.format('bool', key, False)) else: new_opt, misc = self.data_opt_parser.parse_known_args([ '--' + key.replace('_', '-'), str(value) ]) new_value = getattr(new_opt, key) setattr(self.data_opt, key, new_value) print('set {} {}: {}'.format(type(new_value), key, new_value)) except Exception as e: print('error {} - cant set value {}: {}'.format(e, key, value)) def _get_fn(self): return vars(self.data_opt) def _ready_fn(self, rpc_client): # self.connect() process_live_input(self.opt, self.data_opt, rpc_client) def connect(self): self.rpc_client = CortexRPC(self._get_fn, self._set_fn, self._ready_fn) def process_live_input(opt, data_opt, rpc_client): A_offset, A_im, A_dir = load_first_frame(opt, data_opt) data_loader = CreateRecursiveDataLoader(opt) dataset = data_loader.load_data() model = create_model(opt) print("generating...") last_im = None for i, data in enumerate(data_loader): if i >= opt.how_many: break model.set_input(data) model.test() visuals = model.get_current_visuals() img_path = model.get_image_paths() if (i % 100) == 0: print('%04d: process image...' % (i)) im = visuals['fake_B'] last_path = opt.render_dir + "frame_{:05d}.png".format(i) tmp_path = opt.render_dir + "frame_{:05d}_tmp.png".format(i+1) next_path = opt.render_dir + "frame_{:05d}.png".format(i+1) current_path = opt.render_dir + "ren_{:05d}.png".format(i+1) if A_dir is not None: sequence_path = A_dir.format(A_offset+i+1) if data_opt.send_image == 'b': image_pil = Image.fromarray(im, mode='RGB') rpc_client.send_pil_image("frame_{:05d}.png".format(i+1), image_pil) if data_opt.store_a is not True: os.remove(last_path) if data_opt.store_b is True: image_pil = Image.fromarray(im, mode='RGB') image_pil.save(tmp_path) os.rename(tmp_path, current_path) if data_opt.recursive and last_im is not None: if data_opt.sequence and A_dir is not None: A_img = Image.open(sequence_path).convert('RGB') A_im = np.asarray(A_img) frac_a = data_opt.recursive_frac frac_b = data_opt.sequence_frac frac_sum = frac_a + frac_b if frac_sum > 1.0: frac_a = frac_a / frac_sum frac_b = frac_b / frac_sum if data_opt.transition: t = lerp(math.sin(i / data_opt.transition_period * math.pi * 2.0 ) / 2.0 + 0.5, data_opt.transition_min, data_opt.transition_max) frac_a *= 1.0 - t frac_b *= 1.0 - t frac_c = 1.0 - frac_a - frac_b array_a = np.multiply(last_im.astype('float64'), frac_a) array_b = np.multiply(A_im.astype('float64'), frac_b) array_c = np.multiply(im.astype('float64'), frac_c) array_ab = np.add(array_a, array_b) array_abc = np.add(array_ab, array_c) next_im = array_abc.astype('uint8') else: frac_a = data_opt.recursive_frac frac_b = 1.0 - frac_a array_a = np.multiply(last_im.astype('float64'), frac_a) array_b = np.multiply(im.astype('float64'), frac_b) next_im = np.add(array_a, array_b).astype('uint8') if data_opt.recurse_roll != 0: last_im = np.roll(im, data_opt.recurse_roll, axis=data_opt.recurse_roll_axis) else: last_im = next_im.copy().astype('uint8') else: last_im = im.copy().astype('uint8') next_im = im next_img = process_image(opt, data_opt, next_im) if data_opt.send_image == 'sequence': rpc_client.send_pil_image("frame_{:05d}.png".format(i+1), A_img) if data_opt.send_image == 'recursive': pil_im = Image.fromarray(next_im) rpc_client.send_pil_image("frame_{:05d}.png".format(i+1), pil_im) if data_opt.send_image == 'a': rgb_im = cv2.cvtColor(next_img, cv2.COLOR_BGR2RGB) pil_im = Image.fromarray(rgb_im) rpc_client.send_pil_image("frame_{:05d}.png".format(i+1), pil_im) cv2.imwrite(tmp_path, next_img) os.rename(tmp_path, next_path) if data_opt.exit: sys.exit(0) if __name__ == '__main__': listener = Listener() listener.connect()