From 7287a81b9af14ca8b24f258a2e3de08f39bb63c7 Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Tue, 1 May 2018 03:22:09 +0200 Subject: process script --- process_fns | 109 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100755 process_fns (limited to 'process_fns') diff --git a/process_fns b/process_fns new file mode 100755 index 0000000..ad47a28 --- /dev/null +++ b/process_fns @@ -0,0 +1,109 @@ +workon samplernn + +function generate () { + exp_name=$1 + n_samples=$2 + sample_rate=$3 + duration=$4 + let sample_length=$3*$4 + + echo "" + echo "###################################################" + echo "###################################################" + echo "###################################################" + echo "" + echo ">> generating $exp_name" + echo "" + echo "###################################################" + echo "###################################################" + echo "###################################################" + echo "" + python generate.py \ + --exp $exp_name --dataset $exp_name \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --n_samples $n_samples \ + --epoch_limit 1 \ + --resume True + sleep 1 +} +function run () { + exp_name=$1 + epoch_limit=$2 + n_samples=$3 + sample_rate=$4 + duration=$5 + let sample_length=$4*$5 + + echo "" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo "" + echo ">> running $exp_name" + echo "" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo "" + python train.py \ + --exp $exp_name --dataset $exp_name \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --n_samples $n_samples \ + --keep_old_checkpoints False \ + --epoch_limit $epoch_limit \ + --resume True + sleep 1 +} +function standard () { + dataset=$1 + run $1 6 6 44100 5 + ./latest.pl -l $1 +} +function quick () { + dataset=$1 + run $1 4 6 44100 5 + ./latest.pl -l $1 +} +function half () { + dataset=$1 + run $1 2 6 44100 5 + ./latest.pl -l $1 +} +function fast () { + dataset=$1 + run $1 1 6 44100 10 + ./latest.pl -l $1 +} +function emph () { + dataset=$1 + run $1 1 6 44100 10 + run $1 3 6 44100 5 + ./latest.pl -l $1 +} + +function start () { + dataset=$1 + cd datasets/ + perl dataset.pl "${dataset}.wav" + cd .. + quick $dataset +} +function start_standard () { + dataset=$1 + cd datasets/ + perl dataset.pl "${dataset}.wav" + cd .. + standard $dataset +} + -- cgit v1.2.3-70-g09d2 From 0f2cf9df81a6a39aa25be3cca828c4efa47c53a3 Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Sat, 12 May 2018 04:25:55 +0200 Subject: fns --- process_fns | 2 +- upload.js | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'process_fns') diff --git a/process_fns b/process_fns index ad47a28..3964b3b 100755 --- a/process_fns +++ b/process_fns @@ -1,4 +1,4 @@ -workon samplernn +# workon samplernn function generate () { exp_name=$1 diff --git a/upload.js b/upload.js index 73e1147..1abc5e9 100755 --- a/upload.js +++ b/upload.js @@ -32,6 +32,7 @@ var r = request.post('https://bucky.asdf.us/api/thread/' + id + '/comment', func console.log('\n') console.log('Upload successful!') console.log('\n') + console.log(body) argv['_'].forEach(fn => { const partz = fn.split('/') const url = 'https://s3.amazonaws.com/i.asdf.us/bucky/data/' + id + '/' + partz[partz.length-1] -- cgit v1.2.3-70-g09d2 From 3ff94cfea6b8953307e29d5d3ffba9d70cf7659e Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Sat, 12 May 2018 04:28:01 +0200 Subject: gen len --- process_fns | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) (limited to 'process_fns') diff --git a/process_fns b/process_fns index 3964b3b..0bac4e7 100755 --- a/process_fns +++ b/process_fns @@ -32,6 +32,37 @@ function generate () { --resume True sleep 1 } +function gen_len () { + exp_name=$1 + n_samples=$2 + sample_rate=$3 + duration=$4 + sample_length=$5 + + echo "" + echo "###################################################" + echo "###################################################" + echo "###################################################" + echo "" + echo ">> generating $exp_name" + echo "" + echo "###################################################" + echo "###################################################" + echo "###################################################" + echo "" + python generate.py \ + --exp $exp_name --dataset $exp_name \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --n_samples $n_samples \ + --epoch_limit 1 \ + --resume True +} function run () { exp_name=$1 epoch_limit=$2 -- cgit v1.2.3-70-g09d2 From a61412eb1538615578e87a20ddfbc618e1eeb041 Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Sun, 13 May 2018 12:03:37 +0200 Subject: get script now supports urls --- get.pl | 43 ++++++++++++++++++++++++++++++++++++++----- latest.pl | 7 ++++++- process_fns | 3 +-- 3 files changed, 45 insertions(+), 8 deletions(-) (limited to 'process_fns') diff --git a/get.pl b/get.pl index 4447ab3..8a0d8f9 100755 --- a/get.pl +++ b/get.pl @@ -4,11 +4,44 @@ $SIG{TERM} = $SIG{INT} = sub { exit 1 }; chdir('datasets'); -foreach (@ARGV){ - my $s = $_; - if ($s !~/\..*$/) { $s .= ".wav"; } - system('/usr/bin/wget', 'https://neural:spawn5@asdf.us/neural/' . $s); - system('/usr/bin/perl', 'dataset.pl', $s); +my $fn, $new_fn; +foreach my $s (@ARGV){ + if ($s =~ /^http/) { + if ($s =~ /(wav|aiff?|flac|mp3|opus)$/i) { + my $fn = `basename $s`; + system('/usr/bin/wget', $s); + system('/usr/bin/perl', 'dataset.pl', $fn); + } else { + my $yt = `youtube-dl --extract-audio --audio-format flac -o "%(title)s.%(ext)s" $s`; + my @partz = split("\n", $yt); + foreach $part (@partz) { + if ($part =~ /\[ffmpeg\] Destination\: (.*\.flac)$/) { + $fn = $1; + } + } + if ($fn) { + $new_fn = lc $fn; + $new_fn =~ s/\.flac$//g; + $new_fn =~ s/\s+/_/g; + $new_fn =~ s/\W//g; + if (length($new_fn) == 0) { + $new_fn = lc $s; + $new_fn =~ s/\s+/_/g; + $new_fn =~ s/\W//g; + } + $new_fn =~ s/_+/_/g; + $new_fn .= '.flac'; + system('mv', $fn, $new_fn); + print"got fn, $fn => $new_fn\n"; + system('/usr/bin/perl', 'dataset.pl', $new_fn); + } + } + } else { + if ($s !~/\..*$/) { $s .= ".wav"; } + system('/usr/bin/wget', 'https://neural:spawn5@asdf.us/neural/' . $s); + system('/usr/bin/perl', 'dataset.pl', $s); + } + $fn = undef; } END { diff --git a/latest.pl b/latest.pl index ddd5952..1bb7932 100755 --- a/latest.pl +++ b/latest.pl @@ -5,7 +5,8 @@ use Getopt::Std; our $opt_l; # mix and upload files our $opt_v; # print log for all files -getopts('lav'); +our $opt_n; # name/tag for file +getopts('lavn:'); our $matches = 0; if (scalar @ARGV) { @@ -73,6 +74,10 @@ sub process($){ # return 0; #} + if ($opt_l && $opt_n) { + $name .= '-' . $opt_n; + } + if (-e "output/" . $name . ".mp3") { return 1; } diff --git a/process_fns b/process_fns index 0bac4e7..0cbdeee 100755 --- a/process_fns +++ b/process_fns @@ -36,8 +36,7 @@ function gen_len () { exp_name=$1 n_samples=$2 sample_rate=$3 - duration=$4 - sample_length=$5 + sample_length=$4 echo "" echo "###################################################" -- cgit v1.2.3-70-g09d2 From e2d8a6f26c5e44d970d7c069f171105376835495 Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Mon, 14 May 2018 18:57:17 +0200 Subject: log stuff.. auto append --- get.pl | 7 +++++++ process_fns | 11 ++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) (limited to 'process_fns') diff --git a/get.pl b/get.pl index 8a0d8f9..0ef39db 100755 --- a/get.pl +++ b/get.pl @@ -9,9 +9,11 @@ foreach my $s (@ARGV){ if ($s =~ /^http/) { if ($s =~ /(wav|aiff?|flac|mp3|opus)$/i) { my $fn = `basename $s`; + print "downloading $fn\n"; system('/usr/bin/wget', $s); system('/usr/bin/perl', 'dataset.pl', $fn); } else { + print "youtube-dl $s\n"; my $yt = `youtube-dl --extract-audio --audio-format flac -o "%(title)s.%(ext)s" $s`; my @partz = split("\n", $yt); foreach $part (@partz) { @@ -38,9 +40,14 @@ foreach my $s (@ARGV){ } } else { if ($s !~/\..*$/) { $s .= ".wav"; } + print "downloading $s\n"; system('/usr/bin/wget', 'https://neural:spawn5@asdf.us/neural/' . $s); system('/usr/bin/perl', 'dataset.pl', $s); } + open(my $fd, ">>../run_slap.sh"); + print $fd "standard $fn"; + close $fn; + $fn = undef; } diff --git a/process_fns b/process_fns index 0cbdeee..cef738d 100755 --- a/process_fns +++ b/process_fns @@ -1,12 +1,13 @@ # workon samplernn function generate () { - exp_name=$1 + exp_name=${1%.*} n_samples=$2 sample_rate=$3 duration=$4 let sample_length=$3*$4 + sleep 0.1 echo "" echo "###################################################" echo "###################################################" @@ -30,14 +31,14 @@ function generate () { --n_samples $n_samples \ --epoch_limit 1 \ --resume True - sleep 1 } function gen_len () { - exp_name=$1 + exp_name=${1%.*} n_samples=$2 sample_rate=$3 sample_length=$4 + sleep 0.1 echo "" echo "###################################################" echo "###################################################" @@ -63,13 +64,14 @@ function gen_len () { --resume True } function run () { - exp_name=$1 + exp_name=${1%.*} epoch_limit=$2 n_samples=$3 sample_rate=$4 duration=$5 let sample_length=$4*$5 + sleep 0.1 echo "" echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" @@ -93,7 +95,6 @@ function run () { --keep_old_checkpoints False \ --epoch_limit $epoch_limit \ --resume True - sleep 1 } function standard () { dataset=$1 -- cgit v1.2.3-70-g09d2