new fetch/build/train scripts

3 files changed, 143 insertions, 0 deletions

diff --git a/build_dataset.pl b/build_dataset.pl
new file mode 100644
index 0000000..6b0524d
--- /dev/null
+++ b/build_dataset.pl
@@ -0,0 +1,57 @@
+#!/usr/bin/perl
+
+use strict;
+use Cwd qw(cwd);
+my $base_dir = cwd;
+
+our $images_dir = $base_dir . "/sequences/$tag/";
+our $thumbs_dir = $base_dir . "/datasets/$tag/";
+
+mkdir($base_dir . "/datasets/");
+mkdir($thumbs_dir);
+mkdir($thumbs_dir . "train_A/");
+mkdir($thumbs_dir . "test_A/");
+mkdir($thumbs_dir . "train_B/");
+mkdir($thumbs_dir . "test_B/");
+
+our $test_split = 10; # 1/N videos will be for the test split
+
+our @files = ();
+
+opendir DIR, $images_dir;
+while (readdir DIR) {
+  next if /^\./;
+  push(@files, $_);
+}
+closedir DIR;
+
+our @images = sort @files;
+
+my $count = scalar(@images);
+
+print "Processing " . $count . " frames\n";
+
+my $i;
+my $j;
+my $dir;
+my $src;
+my $dest;
+for ($i = 0; $i < $count; $i++) {
+  if (($i % $test_split) != ($test_split-1)) {
+    $dir = $thumbs_dir . "train_";
+  } else {
+    $dir = $thumbs_dir . "test_";
+  }
+
+  $src = $images_dir . $images[$i];
+  $dst = $dir . sprintf("A/frame%03d.png", $i);
+  system("ln", "-s", $src, $dst);
+
+  $src = $images_dir . $images[$i+1];
+  $dst = $dir . sprintf("B/frame%03d.png", $i);
+  system("ln", "-s", $src, $dst);
+
+  if (($i % 100) == 0)
+  print $i . "...\n";
+}
+print "Done\n";
diff --git a/get.pl b/get.pl
new file mode 100755
index 0000000..0a12a81
--- /dev/null
+++ b/get.pl
@@ -0,0 +1,79 @@
+#!/usr/bin/perl
+
+$SIG{TERM} = $SIG{INT} = sub { exit 1 };
+
+sub sanitize ($) {
+  my $s = lc shift;
+  $s =~ s/\.[^.]+$//g;
+  $s =~ s/\W/_/g;
+  $s =~ s/\s+/_/g;
+  if (length($s) == 0) {
+    print "fn is empty";
+    exit(1);
+  }
+  $s =~ s/_+/_/g;
+  return $s;
+}
+
+my ($s) = @ARGV;
+ 
+if (not defined $s || $s !~ /^http/) {
+  die "usage: ./get.pl http://...\n";
+}
+
+mkdir("sequences/");
+chdir("sequences/");
+
+my $fn, $new_fn, $video_fn;
+
+if ($s =~ /(mov|mp4|webm|avi|mpe?g)$/i) {
+  $s =~ s/^\s+//;
+  $s =~ s/\s+$//;
+  my $fn = `basename $s`;
+  $clean_fn = $fn;
+  $clean_fn =~ s/-/_/g;
+  $clean_fn =~ s/^\s//;
+  $clean_fn =~ s/\s$//;
+  $clean_fn =~ s/\s/_/g;
+  $clean_fn =~ s/_+/_/g;
+  system('rm', $fn);
+  system('rm', $clean_fn);
+  print "downloading $clean_fn\n";
+  system('wget', '-O', $clean_fn, $s);
+  $video_fn = $clean_fn;
+}
+else {
+  print "youtube-dl $s\n";
+  my $yt = `youtube-dl -o "%(title)s.%(ext)s" $s`;
+  # print $yt;
+  my @partz = split("\n", $yt);
+  foreach $part (@partz) {
+    print "$part\n";
+    if ($part =~ /\[download\] Destination\: ([^)]+)$/) {
+      $video_fn = $1;
+      print ">>> $video_fn\n";
+    }
+    if ($part =~ /\[download\] (.*) has already been downloaded and merged/) {
+      $video_fn = $1;
+      print ">>> $video_fn\n";
+    }
+    if ($part =~ /\[ffmpeg\] Merging formats into "(.*)"/) {
+      $video_fn = $1;
+      print ">>> $video_fn\n";
+    }
+  }
+}
+
+if ($video_fn) {
+  print "Video filename: $video_fn\n";
+  my $tag = sanitize($video_fn);
+  print "creating dataset: $tag\n";
+  mkdir($tag);
+  system('ffmpeg', '-i', $video_fn, "-filter:v", "scale=1024x512", $tag . '/frame_%05d.png');
+  print "created dataset: $tag\n";
+}
+
+END {
+  chdir('..');
+}
+
diff --git a/train.sh b/train.sh
new file mode 100644
index 0000000..054748e
--- /dev/null
+++ b/train.sh
@@ -0,0 +1,7 @@
+if [ "$1" == "" ]; then
+    echo "Usage: $0 [dataset]"
+    exit 1
+fi
+
+python train.py --name $1 --label_nc 0 --no_instance --continue_train --which_epoch latest
+