From 001c18201e06b0740b8aff7d7a613bb1dcb93553 Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Tue, 10 Apr 2018 02:02:13 +0200 Subject: script to list latest files --- latest.pl | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 latest.pl (limited to 'latest.pl') diff --git a/latest.pl b/latest.pl new file mode 100644 index 0000000..b20ca1c --- /dev/null +++ b/latest.pl @@ -0,0 +1,49 @@ +#!/usr/bin/perl + +use strict; +use Getopt::Std; + +our $opt_l; +getopts('l'); + +# my $s = "results/exp:03_responses-frame_sizes:8,2-n_rnn:2-dataset:03_responses/checkpoints/best-ep1-it1035"; +sub process($){ + my ($dir) = @_; + my @exp = split(":", $dir); + my $name = $exp[4]; + my $ep; + my $path = "results/" . $dir . "/"; + my $chex = $path . "checkpoints/"; + return 0 unless -e $chex; + opendir DIR, $chex or die $!; + while (readdir DIR) { + next unless /^ep/; + $ep = $_; + last; + } + closedir DIR; + + my @epoch = split("-", $ep); + $name .= "-" . $epoch[0]; + + if (-e "output/" . $name . ".mp3") { + return 1; + } + + print $name . "\n"; + if ($opt_l) { + return 0; + } + system('/bin/bash', 'mix.sh', $path, $name); + return 0; +} + +opendir RESULTS, ("results/") or die $!; +my @results = sort {(stat $a)[10] <=> (stat $b)[10]} readdir(RESULTS); +closedir RESULTS; + +for my $result (@results) { + next if $result !~ /exp:/; + my $rv = process($result); + #last if $rv; +} -- cgit v1.2.3-70-g09d2 From 05401dc3db9414986a2a1477e1e5fa08778da45c Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Tue, 10 Apr 2018 10:45:47 +0200 Subject: latest renders with -l, get.pl to wget from asdf.us --- datasets/get.pl | 4 ++++ latest.pl | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100755 datasets/get.pl (limited to 'latest.pl') diff --git a/datasets/get.pl b/datasets/get.pl new file mode 100755 index 0000000..8b821c6 --- /dev/null +++ b/datasets/get.pl @@ -0,0 +1,4 @@ +#!/usr/bin/perl + +foreach (@ARGV){ system('/usr/bin/wget', 'https://asdf.us/neural/' . $_); } + diff --git a/latest.pl b/latest.pl index b20ca1c..1e3a46e 100644 --- a/latest.pl +++ b/latest.pl @@ -26,14 +26,20 @@ sub process($){ my @epoch = split("-", $ep); $name .= "-" . $epoch[0]; + if (! $epoch[0]) { + return 0; + } if (-e "output/" . $name . ".mp3") { return 1; } print $name . "\n"; - if ($opt_l) { + if (!$opt_l) { return 0; } + print "\n"; + print "_______________________________________________________________\n"; + print "\n"; system('/bin/bash', 'mix.sh', $path, $name); return 0; } -- cgit v1.2.3-70-g09d2 From f3d8faecdcf0de97e8027ca9d80315b58fab700a Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Tue, 10 Apr 2018 15:10:24 +0200 Subject: ignore results that are insufficiently trained --- latest.pl | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'latest.pl') diff --git a/latest.pl b/latest.pl index 1e3a46e..f1d1aa9 100644 --- a/latest.pl +++ b/latest.pl @@ -29,6 +29,10 @@ sub process($){ if (! $epoch[0]) { return 0; } + $epoch[0] =~ /(\d+)/; + if ($1 < 4) { + return 0; + } if (-e "output/" . $name . ".mp3") { return 1; } -- cgit v1.2.3-70-g09d2 From 3ea2f8f581a857920b7c2f4d1d3a0a12401758ec Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Tue, 10 Apr 2018 15:20:00 +0200 Subject: build --- latest.pl | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) (limited to 'latest.pl') diff --git a/latest.pl b/latest.pl index f1d1aa9..33bfb73 100644 --- a/latest.pl +++ b/latest.pl @@ -23,6 +23,20 @@ sub process($){ } closedir DIR; + my $sampz = $path . "samples/"; + return 0 unless -e $sampz; + my $c = 0; + opendir DIR, $sampz or die $!; + while (readdir DIR) { + next unless /wav$/; + $c += 1; + last; + } + closedir DIR; + if ($c == 0) { + return 0; + } + my @epoch = split("-", $ep); $name .= "-" . $epoch[0]; @@ -37,13 +51,14 @@ sub process($){ return 1; } - print $name . "\n"; if (!$opt_l) { return 0; } print "\n"; print "_______________________________________________________________\n"; print "\n"; + print $name . ", $c samples\n"; + print "\n"; system('/bin/bash', 'mix.sh', $path, $name); return 0; } -- cgit v1.2.3-70-g09d2 From 332d6f5b739c8e037e3d2877085f45fbeef375e8 Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Tue, 10 Apr 2018 15:57:54 +0200 Subject: get script should build dataset --- datasets/get.pl | 6 +++++- latest.pl | 0 2 files changed, 5 insertions(+), 1 deletion(-) mode change 100644 => 100755 latest.pl (limited to 'latest.pl') diff --git a/datasets/get.pl b/datasets/get.pl index 531b9f7..14c5a5b 100755 --- a/datasets/get.pl +++ b/datasets/get.pl @@ -1,5 +1,9 @@ #!/usr/bin/perl foreach (@ARGV){ - my $s = $_; if ($s !~/\..*$/) { $s .= ".wav"; } system('/usr/bin/wget', 'https://neural:spawn5@asdf.us/neural/' . $s); } + my $s = $_; + if ($s !~/\..*$/) { $s .= ".wav"; } + system('/usr/bin/wget', 'https://neural:spawn5@asdf.us/neural/' . $s); + system('/usr/bin/perl', 'dataset.pl', $s); +} diff --git a/latest.pl b/latest.pl old mode 100644 new mode 100755 -- cgit v1.2.3-70-g09d2 From b8ce4da58bef73ce4adb7c278803fcef37b9c2ad Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Tue, 10 Apr 2018 16:29:11 +0200 Subject: latest --- latest.pl | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'latest.pl') diff --git a/latest.pl b/latest.pl index 33bfb73..cd86de2 100755 --- a/latest.pl +++ b/latest.pl @@ -3,8 +3,9 @@ use strict; use Getopt::Std; -our $opt_l; -getopts('l'); +our $opt_l; # mix and upload files +our $opt_v; # print log for all files +getopts('lav'); # my $s = "results/exp:03_responses-frame_sizes:8,2-n_rnn:2-dataset:03_responses/checkpoints/best-ep1-it1035"; sub process($){ @@ -14,6 +15,7 @@ sub process($){ my $ep; my $path = "results/" . $dir . "/"; my $chex = $path . "checkpoints/"; + my $log = $path . 'log'; return 0 unless -e $chex; opendir DIR, $chex or die $!; while (readdir DIR) { @@ -47,6 +49,11 @@ sub process($){ if ($1 < 4) { return 0; } + + if ($opt_v) { + print `grep valid $log`; + } + if (-e "output/" . $name . ".mp3") { return 1; } -- cgit v1.2.3-70-g09d2 From a1c16f063f7d753909c75ca9cbac459af7eb8245 Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Tue, 10 Apr 2018 16:29:50 +0200 Subject: latest --- latest.pl | 1 + 1 file changed, 1 insertion(+) (limited to 'latest.pl') diff --git a/latest.pl b/latest.pl index cd86de2..2218d06 100755 --- a/latest.pl +++ b/latest.pl @@ -51,6 +51,7 @@ sub process($){ } if ($opt_v) { + print $name; print `grep valid $log`; } -- cgit v1.2.3-70-g09d2 From 48f321017c86097d7500318f748a90959f5fafba Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Tue, 10 Apr 2018 16:30:45 +0200 Subject: latest --- latest.pl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'latest.pl') diff --git a/latest.pl b/latest.pl index 2218d06..a29cf50 100755 --- a/latest.pl +++ b/latest.pl @@ -35,6 +35,12 @@ sub process($){ last; } closedir DIR; + + if ($opt_v) { + print $name . "\n"; + print `grep valid $log`; + } + if ($c == 0) { return 0; } @@ -50,11 +56,6 @@ sub process($){ return 0; } - if ($opt_v) { - print $name; - print `grep valid $log`; - } - if (-e "output/" . $name . ".mp3") { return 1; } -- cgit v1.2.3-70-g09d2 From fb5da04e2ec639c8791a400dd9a748de1484d89f Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Tue, 10 Apr 2018 16:32:20 +0200 Subject: stat 9 --- latest.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'latest.pl') diff --git a/latest.pl b/latest.pl index a29cf50..f1a36a2 100755 --- a/latest.pl +++ b/latest.pl @@ -73,7 +73,7 @@ sub process($){ } opendir RESULTS, ("results/") or die $!; -my @results = sort {(stat $a)[10] <=> (stat $b)[10]} readdir(RESULTS); +my @results = sort {(stat $a)[9] <=> (stat $b)[9]} readdir(RESULTS); closedir RESULTS; for my $result (@results) { -- cgit v1.2.3-70-g09d2 From 7c0c0850ed7de27e42594b1862ba18e55cbe4221 Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Tue, 10 Apr 2018 16:34:01 +0200 Subject: latest --- latest.pl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'latest.pl') diff --git a/latest.pl b/latest.pl index f1a36a2..6fd6dcd 100755 --- a/latest.pl +++ b/latest.pl @@ -73,7 +73,8 @@ sub process($){ } opendir RESULTS, ("results/") or die $!; -my @results = sort {(stat $a)[9] <=> (stat $b)[9]} readdir(RESULTS); +# my @results = sort {(stat $a)[9] <=> (stat $b)[9]} readdir(RESULTS); +my @results = sort {$a cmp $b} readdir(RESULTS); closedir RESULTS; for my $result (@results) { -- cgit v1.2.3-70-g09d2 From 860acac9862cd5ebe80d2a428d446dbdcc41d197 Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Tue, 10 Apr 2018 16:34:41 +0200 Subject: latest --- latest.pl | 1 + 1 file changed, 1 insertion(+) (limited to 'latest.pl') diff --git a/latest.pl b/latest.pl index 6fd6dcd..60d4a06 100755 --- a/latest.pl +++ b/latest.pl @@ -39,6 +39,7 @@ sub process($){ if ($opt_v) { print $name . "\n"; print `grep valid $log`; + print "\n"; } if ($c == 0) { -- cgit v1.2.3-70-g09d2 From 805a4fc2b3e406e0b1506fdfd54aa8750d289659 Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Tue, 10 Apr 2018 17:49:13 +0200 Subject: counting samples again --- latest.pl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'latest.pl') diff --git a/latest.pl b/latest.pl index 60d4a06..4975a0a 100755 --- a/latest.pl +++ b/latest.pl @@ -32,7 +32,6 @@ sub process($){ while (readdir DIR) { next unless /wav$/; $c += 1; - last; } closedir DIR; @@ -52,6 +51,11 @@ sub process($){ if (! $epoch[0]) { return 0; } + + if (! $opt_l) { + print $name . ", $c samples\n"; + } + $epoch[0] =~ /(\d+)/; if ($1 < 4) { return 0; -- cgit v1.2.3-70-g09d2 From a04b7b8117ee6993bba35ea99fbd9f2530213e2d Mon Sep 17 00:00:00 2001 From: jules Date: Wed, 11 Apr 2018 11:26:32 +0200 Subject: specify which experiment to process --- latest.pl | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'latest.pl') diff --git a/latest.pl b/latest.pl index 4975a0a..c09a5cd 100755 --- a/latest.pl +++ b/latest.pl @@ -7,6 +7,14 @@ our $opt_l; # mix and upload files our $opt_v; # print log for all files getopts('lav'); +our $matches = 0; +if (scalar @ARGV) { + $matches = {}; + for (@ARGV) { + my ($name, $ep) = split("-", $_, 2); + $matches->{$name} = 1; + } +} # my $s = "results/exp:03_responses-frame_sizes:8,2-n_rnn:2-dataset:03_responses/checkpoints/best-ep1-it1035"; sub process($){ my ($dir) = @_; @@ -35,6 +43,10 @@ sub process($){ } closedir DIR; + if ($matches && ! exists $matches->{$name}) { + return 0; + } + if ($opt_v) { print $name . "\n"; print `grep valid $log`; -- cgit v1.2.3-70-g09d2 From 7287a81b9af14ca8b24f258a2e3de08f39bb63c7 Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Tue, 1 May 2018 03:22:09 +0200 Subject: process script --- .gitignore | 3 ++ datasets/dataset.pl | 4 ++ datasets/silence.sh | 0 latest.pl | 6 +-- process_fns | 109 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 119 insertions(+), 3 deletions(-) create mode 100644 datasets/silence.sh create mode 100755 process_fns (limited to 'latest.pl') diff --git a/.gitignore b/.gitignore index 2ad71be..26a1d76 100644 --- a/.gitignore +++ b/.gitignore @@ -110,3 +110,6 @@ node_modules/ results/old_checkpoints/ +mgb +run_* + diff --git a/datasets/dataset.pl b/datasets/dataset.pl index edde43f..a6eb984 100755 --- a/datasets/dataset.pl +++ b/datasets/dataset.pl @@ -25,6 +25,10 @@ sub process($) { return; } + if ($ext eq 'mp3') { + system('ffmpeg', '-i', $filename, $name . '.wav') + $filename = $name . '.wav'; + } my $soxi = `soxi $filename`; my @lines = split("\n", $soxi); diff --git a/datasets/silence.sh b/datasets/silence.sh new file mode 100644 index 0000000..e69de29 diff --git a/latest.pl b/latest.pl index c09a5cd..ddd5952 100755 --- a/latest.pl +++ b/latest.pl @@ -69,9 +69,9 @@ sub process($){ } $epoch[0] =~ /(\d+)/; - if ($1 < 4) { - return 0; - } + #if ($1 < 4) { + # return 0; + #} if (-e "output/" . $name . ".mp3") { return 1; diff --git a/process_fns b/process_fns new file mode 100755 index 0000000..ad47a28 --- /dev/null +++ b/process_fns @@ -0,0 +1,109 @@ +workon samplernn + +function generate () { + exp_name=$1 + n_samples=$2 + sample_rate=$3 + duration=$4 + let sample_length=$3*$4 + + echo "" + echo "###################################################" + echo "###################################################" + echo "###################################################" + echo "" + echo ">> generating $exp_name" + echo "" + echo "###################################################" + echo "###################################################" + echo "###################################################" + echo "" + python generate.py \ + --exp $exp_name --dataset $exp_name \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --n_samples $n_samples \ + --epoch_limit 1 \ + --resume True + sleep 1 +} +function run () { + exp_name=$1 + epoch_limit=$2 + n_samples=$3 + sample_rate=$4 + duration=$5 + let sample_length=$4*$5 + + echo "" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo "" + echo ">> running $exp_name" + echo "" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo "" + python train.py \ + --exp $exp_name --dataset $exp_name \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --n_samples $n_samples \ + --keep_old_checkpoints False \ + --epoch_limit $epoch_limit \ + --resume True + sleep 1 +} +function standard () { + dataset=$1 + run $1 6 6 44100 5 + ./latest.pl -l $1 +} +function quick () { + dataset=$1 + run $1 4 6 44100 5 + ./latest.pl -l $1 +} +function half () { + dataset=$1 + run $1 2 6 44100 5 + ./latest.pl -l $1 +} +function fast () { + dataset=$1 + run $1 1 6 44100 10 + ./latest.pl -l $1 +} +function emph () { + dataset=$1 + run $1 1 6 44100 10 + run $1 3 6 44100 5 + ./latest.pl -l $1 +} + +function start () { + dataset=$1 + cd datasets/ + perl dataset.pl "${dataset}.wav" + cd .. + quick $dataset +} +function start_standard () { + dataset=$1 + cd datasets/ + perl dataset.pl "${dataset}.wav" + cd .. + standard $dataset +} + -- cgit v1.2.3-70-g09d2 From a61412eb1538615578e87a20ddfbc618e1eeb041 Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Sun, 13 May 2018 12:03:37 +0200 Subject: get script now supports urls --- get.pl | 43 ++++++++++++++++++++++++++++++++++++++----- latest.pl | 7 ++++++- process_fns | 3 +-- 3 files changed, 45 insertions(+), 8 deletions(-) (limited to 'latest.pl') diff --git a/get.pl b/get.pl index 4447ab3..8a0d8f9 100755 --- a/get.pl +++ b/get.pl @@ -4,11 +4,44 @@ $SIG{TERM} = $SIG{INT} = sub { exit 1 }; chdir('datasets'); -foreach (@ARGV){ - my $s = $_; - if ($s !~/\..*$/) { $s .= ".wav"; } - system('/usr/bin/wget', 'https://neural:spawn5@asdf.us/neural/' . $s); - system('/usr/bin/perl', 'dataset.pl', $s); +my $fn, $new_fn; +foreach my $s (@ARGV){ + if ($s =~ /^http/) { + if ($s =~ /(wav|aiff?|flac|mp3|opus)$/i) { + my $fn = `basename $s`; + system('/usr/bin/wget', $s); + system('/usr/bin/perl', 'dataset.pl', $fn); + } else { + my $yt = `youtube-dl --extract-audio --audio-format flac -o "%(title)s.%(ext)s" $s`; + my @partz = split("\n", $yt); + foreach $part (@partz) { + if ($part =~ /\[ffmpeg\] Destination\: (.*\.flac)$/) { + $fn = $1; + } + } + if ($fn) { + $new_fn = lc $fn; + $new_fn =~ s/\.flac$//g; + $new_fn =~ s/\s+/_/g; + $new_fn =~ s/\W//g; + if (length($new_fn) == 0) { + $new_fn = lc $s; + $new_fn =~ s/\s+/_/g; + $new_fn =~ s/\W//g; + } + $new_fn =~ s/_+/_/g; + $new_fn .= '.flac'; + system('mv', $fn, $new_fn); + print"got fn, $fn => $new_fn\n"; + system('/usr/bin/perl', 'dataset.pl', $new_fn); + } + } + } else { + if ($s !~/\..*$/) { $s .= ".wav"; } + system('/usr/bin/wget', 'https://neural:spawn5@asdf.us/neural/' . $s); + system('/usr/bin/perl', 'dataset.pl', $s); + } + $fn = undef; } END { diff --git a/latest.pl b/latest.pl index ddd5952..1bb7932 100755 --- a/latest.pl +++ b/latest.pl @@ -5,7 +5,8 @@ use Getopt::Std; our $opt_l; # mix and upload files our $opt_v; # print log for all files -getopts('lav'); +our $opt_n; # name/tag for file +getopts('lavn:'); our $matches = 0; if (scalar @ARGV) { @@ -73,6 +74,10 @@ sub process($){ # return 0; #} + if ($opt_l && $opt_n) { + $name .= '-' . $opt_n; + } + if (-e "output/" . $name . ".mp3") { return 1; } diff --git a/process_fns b/process_fns index 0bac4e7..0cbdeee 100755 --- a/process_fns +++ b/process_fns @@ -36,8 +36,7 @@ function gen_len () { exp_name=$1 n_samples=$2 sample_rate=$3 - duration=$4 - sample_length=$5 + sample_length=$4 echo "" echo "###################################################" -- cgit v1.2.3-70-g09d2