new builder scripts

7 files changed, 380 insertions, 7 deletions

diff --git a/scripts/builders/canny-dataset.pl b/scripts/builders/canny-dataset.pl
new file mode 100644
index 0000000..6d1085d
--- /dev/null
+++ b/scripts/builders/canny-dataset.pl
@@ -0,0 +1,39 @@
+import os
+from shutil import move, copyfile
+from PIL import Image, ImageOps
+from shutil import copyfile, rmtree
+import numpy as np
+import cv2
+
+work_dir = "woodsized/"
+render_dir = "woodcanny/"
+
+if os.path.exists(render_dir):
+    rmtree(render_dir)
+
+os.makedirs(render_dir)
+os.makedirs(render_dir + "A/")
+os.makedirs(render_dir + "A/train/")
+os.makedirs(render_dir + "A/test/")
+os.makedirs(render_dir + "A/val/")
+os.makedirs(render_dir + "B/")
+os.makedirs(render_dir + "B/train/")
+os.makedirs(render_dir + "B/test/")
+os.makedirs(render_dir + "B/val/")
+
+for i,fn in enumerate(sorted(os.listdir(work_dir))):
+    pil_image = Image.open(work_dir + fn).convert('RGB')
+    opencv_image = np.array(pil_image)
+    opencv_image = opencv_image[:, :, ::-1].copy()
+    opencv_image = cv2.GaussianBlur(opencv_image, (3,3), 1)
+    opencv_image = cv2.Canny(opencv_image, 100, 200)
+    ren = "frame_{:05d}.png".format(i)
+    if (i % 10) == 3:
+      wd = "test/"
+    elif (i % 10) == 6:
+      wd = "val/"
+    else:
+      wd = "train/"
+    cv2.imwrite(render_dir + "A/" + wd + ren, opencv_image)
+    copyfile(work_dir + fn, render_dir + "B/" + wd + ren)
+
diff --git a/scripts/builders/dataset-random-skip.pl b/scripts/builders/dataset-random-skip.pl
new file mode 100755
index 0000000..bc59723
--- /dev/null
+++ b/scripts/builders/dataset-random-skip.pl
@@ -0,0 +1,64 @@
+#!/usr/bin/perl
+
+use strict;
+
+our $dt = 4;
+our $count = 30000;
+
+our $images_dir = "woodtripod/";
+our $thumbs_dir = "thumbs/randomcrops_" . $count . "_" . $dt . "/";
+
+mkdir($thumbs_dir);
+mkdir($thumbs_dir . "A/");
+mkdir($thumbs_dir . "B/");
+mkdir($thumbs_dir . "A/train/");
+mkdir($thumbs_dir . "B/train/");
+mkdir($thumbs_dir . "A/val/");
+mkdir($thumbs_dir . "B/val/");
+mkdir($thumbs_dir . "A/test/");
+mkdir($thumbs_dir . "B/test/");
+
+our @files = ();
+
+opendir DIR, $images_dir;
+while (readdir DIR) {
+  next if /^\./;
+  push(@files, $_);
+}
+closedir DIR;
+
+our @images = sort @files;
+
+# assume all images are same size
+our $w = 1920;
+our $h = 1080;
+
+our $dw = 256;
+our $dh = 256;
+
+my $i; my $x; my $y; my $dir;
+for ($i = 0; $i < $count; $i++) {
+  my $id = int($i/$count * (scalar @images - $dt));
+  if ( ($id % 7) == 3) {
+    $dir = "test/";
+  } elsif ( ($id % 7) == 6) {
+    $dir = "val/";
+  } else {
+    $dir = "train/";
+  }
+  if ( $i && ($i % 1000) == 0) {
+    print($id . "...\n")
+  }
+
+  my $x = int(rand($w - $dw));
+  my $y = int(rand($h - $dh));
+  my $a_frame = $images_dir . $images[$id];
+  my $b_frame = $images_dir . $images[$id+$dt];
+  my $crop = $dw . "x" . $dh . "+" . $x . "x" . $y;
+  my $fn = sprintf("frame_%04d.png", $i);
+  print "$fn $x $y $a_frame\n";
+  system("convert", $a_frame, "-crop", $crop, $thumbs_dir . "A/" . $dir . $fn);
+  system("convert", $b_frame, "-crop", $crop, $thumbs_dir . "B/" . $dir . $fn);
+}
+print $count . "\n";
+
diff --git a/scripts/builders/dataset-random.pl b/scripts/builders/dataset-random.pl
new file mode 100755
index 0000000..c02c32b
--- /dev/null
+++ b/scripts/builders/dataset-random.pl
@@ -0,0 +1,63 @@
+#!/usr/bin/perl
+
+use strict;
+
+our $images_dir = "/home/lens/Desktop/woodtripod/";
+our $thumbs_dir = "/home/lens/Desktop/thumbs/randomcrops5k/";
+
+our $count = 5000;
+
+mkdir($thumbs_dir);
+mkdir($thumbs_dir . "A/");
+mkdir($thumbs_dir . "B/");
+mkdir($thumbs_dir . "A/train/");
+mkdir($thumbs_dir . "B/train/");
+mkdir($thumbs_dir . "A/val/");
+mkdir($thumbs_dir . "B/val/");
+mkdir($thumbs_dir . "A/test/");
+mkdir($thumbs_dir . "B/test/");
+
+our @files = ();
+
+opendir DIR, $images_dir;
+while (readdir DIR) {
+  next if /^\./;
+  push(@files, $_);
+}
+closedir DIR;
+
+our @images = sort @files;
+
+# assume all images are same size
+our $w = 1920;
+our $h = 1080;
+
+our $dw = 256;
+our $dh = 256;
+
+my $i; my $x; my $y; my $dir;
+for ($i = 0; $i < $count; $i++) {
+  my $id = int($i/$count * (scalar @images - 1));
+  if ( ($id % 7) == 3) {
+    $dir = "test/";
+  } elsif ( ($id % 7) == 6) {
+    $dir = "val/";
+  } else {
+    $dir = "train/";
+  }
+  if ( $i && ($i % 1000) == 0) {
+    print($id . "...\n")
+  }
+
+  my $x = int(rand($w - $dw));
+  my $y = int(rand($h - $dh));
+  my $a_frame = $images_dir . $images[$id];
+  my $b_frame = $images_dir . $images[$id+1];
+  my $crop = $dw . "x" . $dh . "+" . $x . "x" . $y;
+  my $fn = sprintf("frame_%04d.png", $i);
+  print "$fn $x $y $a_frame\n";
+  system("convert", $a_frame, "-crop", $crop, $thumbs_dir . "A/" . $dir . $fn);
+  system("convert", $b_frame, "-crop", $crop, $thumbs_dir . "B/" . $dir . $fn);
+}
+print $count . "\n";
+
diff --git a/scripts/builders/flat-dataset.pl b/scripts/builders/flat-dataset.pl
new file mode 100755
index 0000000..48479bd
--- /dev/null
+++ b/scripts/builders/flat-dataset.pl
@@ -0,0 +1,58 @@
+#!/usr/bin/perl
+
+use strict;
+
+# Processing script for Adversarial Video Generation library
+# Images should all be 320x or similar
+
+our $images_dir = "/home/lens/Desktop/woods_videos/kaulsdorf_large/";
+our $thumbs_dir = "/home/lens/Desktop/thumbs/kaulsdorf_large/";
+
+mkdir($thumbs_dir);
+mkdir($thumbs_dir . "Train/");
+mkdir($thumbs_dir . "Test/");
+
+our $frames_per_dir = 64;
+our $test_split = 5; # 1/N videos will be for the test split
+
+our @files = ();
+
+opendir DIR, $images_dir;
+while (readdir DIR) {
+  next if /^\./;
+  push(@files, $_);
+}
+closedir DIR;
+
+our @images = sort @files;
+
+my $count = scalar(@images);
+
+my $i;
+my $j;
+my $dirid = -2;
+my $nextid;
+my $dir;
+for ($i = 0; $i < $count; $i++) {
+  $nextid = int($i / $frames_per_dir);
+  if ($nextid != $dirid) {
+    $dirid = $nextid;
+    if (($dirid % $test_split) != ($test_split-1)) {
+      $dir = $thumbs_dir . "Train/video" . $dirid . "/";
+    } else {
+      $dir = $thumbs_dir . "Test/video" . $dirid . "/";
+    }
+    print $dir . "\n";
+    mkdir($dir);
+    $j = 0;
+  } else {
+    $j += 1;
+  }
+
+  my $src = $images_dir . $images[$i];
+  my $dst = $dir . sprintf("frame%03d.png", $j);
+  system("cp", $src, $dst);
+  print $dst . "\n";
+}
+print $count . "\n";
+
diff --git a/scripts/builders/flat-multi-dataset.pl b/scripts/builders/flat-multi-dataset.pl
new file mode 100755
index 0000000..a111983
--- /dev/null
+++ b/scripts/builders/flat-multi-dataset.pl
@@ -0,0 +1,75 @@
+#!/usr/bin/perl
+
+use strict;
+
+# Processing script for Adversarial Video Generation library
+# Images should all be 320x or similar
+
+our $images_dir = "/home/lens/Desktop/woods_gray/";
+our $thumbs_dir = "/home/lens/Desktop/thumbs/kaulsdorf_gray/";
+
+mkdir($thumbs_dir);
+mkdir($thumbs_dir . "Train/");
+mkdir($thumbs_dir . "Test/");
+
+our $frames_per_dir = 256;
+our $test_split = 5; # 1/N videos will be for the test split
+
+our $dircount = 0;
+
+sub parse_dir ($) {
+  our ($im_dir) = @_;
+  our @files = ();
+
+  print $im_dir . "\n";
+
+  opendir DIR, $im_dir;
+  while (readdir DIR) {
+    next if /^\./;
+    push(@files, $_);
+  }
+  closedir DIR;
+
+  our @images = sort @files;
+
+  my $count = scalar(@images);
+
+  my $i;
+  my $j;
+  my $dirid = -2;
+  my $nextid;
+  my $dir;
+  for ($i = 0; $i < $count; $i++) {
+    $nextid = int($i / $frames_per_dir) + $dircount;
+    if ($nextid != $dirid) {
+      $dirid = $nextid;
+      if (($dirid % $test_split) != ($test_split-1)) {
+        $dir = $thumbs_dir . "Train/video" . $dirid . "/";
+      } else {
+        $dir = $thumbs_dir . "Test/video" . $dirid . "/";
+      }
+      print $dir . "\n";
+      mkdir($dir);
+      $j = 0;
+    } else {
+      $j += 1;
+    }
+
+    my $src = $im_dir . $images[$i];
+    my $dst = $dir . sprintf("frame%03d.png", $j);
+    system("cp", $src, $dst);
+    # print $dst . "\n";
+  }
+
+  $dircount = $nextid + 1;
+  print $count . " " . $dircount . "\n";
+}
+
+opendir D, $images_dir;
+while (readdir D) {
+  next if /^\./;
+  next if ! -d $images_dir . $_;
+  parse_dir($images_dir . $_ . "/");
+}
+closedir D;
+
diff --git a/scripts/builders/pair-dataset-copy.pl b/scripts/builders/pair-dataset-copy.pl
new file mode 100755
index 0000000..2721aa5
--- /dev/null
+++ b/scripts/builders/pair-dataset-copy.pl
@@ -0,0 +1,65 @@
+#!/usr/bin/perl
+
+use strict;
+
+our $dataset = $ARGV[0];
+print $dataset . "\n";
+if (! $dataset || ! -e $dataset) {
+  print "usage: $0 [name-of-dataset]\n";
+  exit(1);
+}
+
+our $images_dir = "$ENV{'HOME'}/Desktop/$dataset/";
+our $thumbs_dir = "$ENV{'HOME'}/Desktop/thumbs/$dataset/";
+
+our $dt = 1;
+
+mkdir($thumbs_dir);
+mkdir($thumbs_dir . "A/");
+mkdir($thumbs_dir . "B/");
+mkdir($thumbs_dir . "A/train/");
+mkdir($thumbs_dir . "B/train/"); mkdir($thumbs_dir . "A/val/");
+mkdir($thumbs_dir . "B/val/");
+mkdir($thumbs_dir . "A/test/");
+mkdir($thumbs_dir . "B/test/");
+
+our @files = ();
+
+opendir DIR, $images_dir;
+while (readdir DIR) {
+  next if /^\./;
+  push(@files, $_);
+}
+closedir DIR;
+
+our @images = sort @files;
+
+my $count = scalar(@images) - $dt;
+
+print "$count images\n";
+
+my $i; my $x; my $y; my $dir; my $id;
+for ($i = 0; $i < $count; $i++) {
+  $id = $i;
+  if ( ($id % 10) == 5) {
+    $dir = "test/";
+  } elsif ( ($id % 10) == 8) {
+    $dir = "val/";
+  } else {
+    $dir = "train/";
+  }
+  if ( $i && ($i % 1000) == 0) {
+    print($id . "...\n")
+  }
+
+  #my $x = int(rand($w - $dw));
+  #my $y = int(rand($h - $dh));
+  my $a_frame = $images_dir . $images[$id];
+  my $b_frame = $images_dir . $images[$id+$dt];
+  my $fn = sprintf("frame_%05d.png", $i);
+  #print "$fn $x $y $a_frame\n";
+  system("cp", $a_frame, $thumbs_dir . "A/" . $dir . $fn);
+  system("cp", $b_frame, $thumbs_dir . "B/" . $dir . $fn);
+}
+print $count . "\n";
+
diff --git a/scripts/builders/pair-dataset.pl b/scripts/builders/pair-dataset.pl
index 06c40c0..100cf9f 100644
--- a/scripts/builders/pair-dataset.pl
+++ b/scripts/builders/pair-dataset.pl
@@ -2,8 +2,8 @@
 
 use strict;
 
-our $images_dir = "/home/ubuntu/Desktop/wood/";
-our $thumbs_dir = "/home/ubuntu/Desktop/thumbs/wood/";
+our $images_dir = "/home/lens/Desktop/messi/";
+our $thumbs_dir = "/home/lens/Desktop/thumbs/messi/";
 
 our $dt = 1;
 
@@ -11,8 +11,7 @@ mkdir($thumbs_dir);
 mkdir($thumbs_dir . "A/");
 mkdir($thumbs_dir . "B/");
 mkdir($thumbs_dir . "A/train/");
-mkdir($thumbs_dir . "B/train/");
-mkdir($thumbs_dir . "A/val/");
+mkdir($thumbs_dir . "B/train/"); mkdir($thumbs_dir . "A/val/");
 mkdir($thumbs_dir . "B/val/");
 mkdir($thumbs_dir . "A/test/");
 mkdir($thumbs_dir . "B/test/");
@@ -28,7 +27,14 @@ closedir DIR;
 
 our @images = sort @files;
 
-my $count = scalar(@images) - $dt;
+my $count = 3500; # scalar(@images) - $dt;
+
+# assume all images are same size
+our $w = 1920;
+our $h = 1080;
+
+our $dw = 256;
+our $dh = 256;
 
 my $i; my $x; my $y; my $dir;
 for ($i = 0; $i < $count; $i++) {
@@ -44,11 +50,14 @@ for ($i = 0; $i < $count; $i++) {
     print($id . "...\n")
   }
 
+  #my $x = int(rand($w - $dw));
+  #my $y = int(rand($h - $dh));
   my $a_frame = $images_dir . $images[$id];
   my $b_frame = $images_dir . $images[$id+$dt];
-  my $fn = sprintf("frame_%05d.png", $i);
+  my $crop = $dw . "x" . $dh . "+" . $x . "x" . $y;
+  my $fn = sprintf("frame_%04d.png", $i);
   #print "$fn $x $y $a_frame\n";
-  system("convert", $a_frame, "-resize", '256x256!', '-canny', '0x1+10%+30%', $thumbs_dir . "A/" . $dir . $fn);
+  system("convert", $a_frame, "-resize", '256x256!', $thumbs_dir . "A/" . $dir . $fn);
   system("convert", $b_frame, "-resize", '256x256!', $thumbs_dir . "B/" . $dir . $fn);
 }
 print $count . "\n";