diff options
| author | Jules Laplace <julescarbon@gmail.com> | 2018-05-14 19:15:58 +0200 |
|---|---|---|
| committer | Jules Laplace <julescarbon@gmail.com> | 2018-05-14 19:15:58 +0200 |
| commit | 60fb2b7c87b7e6aa179c6a973a8d6e39cbe7c594 (patch) | |
| tree | 8a738a43e8583f38f151cdc643a38b5a9437cda2 | |
| parent | 9766ef0f3a539be7ee68bb93918f25a3298afe39 (diff) | |
| parent | e2d8a6f26c5e44d970d7c069f171105376835495 (diff) | |
Merge branch 'master' of asdf.us:samplernn
39 files changed, 3177 insertions, 5 deletions
@@ -92,3 +92,24 @@ ENV/ *~ *.swp *.swo + +*.wav +*.mp3 +*.aif +*.aiff +*.flac + +results/exp* +!results/*.sh + +run_here* + +.env + +node_modules/ + +results/old_checkpoints/ + +mgb +run_* + diff --git a/best-epoch.sh b/best-epoch.sh new file mode 100755 index 0000000..230daa0 --- /dev/null +++ b/best-epoch.sh @@ -0,0 +1 @@ +ls -lat results/*/checkpoints/best* | head -n 20 diff --git a/bin/crossfade_cat.sh b/bin/crossfade_cat.sh new file mode 100755 index 0000000..13a8e90 --- /dev/null +++ b/bin/crossfade_cat.sh @@ -0,0 +1,129 @@ +#!/bin/bash +# +# crossfade_cat.sh +# +# Concatenates two files together with a crossfade of $1 seconds. +# Filenames are specified as $2 and $3. +# +# $4 is optional and specifies if a fadeout should be performed on +# first file. +# $5 is optional and specifies if a fadein should be performed on +# second file. +# +# Example: crossfade_cat.sh 10 infile1.wav infile2.wav auto auto +# +# By default, the script attempts to guess if the audio files +# already have a fadein/out on them or if they just have really +# low volumes that won't cause clipping when mixxing. If this +# is not detected then the script will perform a fade in/out to +# prevent clipping. +# +# The user may specify "yes" or "no" to force the fade in/out +# to occur. They can also specify "auto" which is the default. +# +# Crossfaded file is created as "mix.wav". +# +# Original script from Kester Clegg. Mods by Chris Bagwell to show +# more examples of sox features. +# + +SOX=sox +SOXI=soxi + +if [ "$3" == "" ]; then + echo "Usage: $0 crossfade_seconds first_file second_file [ fadeout ] [ fadein ]" + echo + echo "If a fadeout or fadein is not desired then specify \"no\" for that option. \"yes\" will force a fade and \"auto\" will try to detect if a fade should occur." + echo + echo "Example: $0 10 infile1.wav infile2.wav auto auto" + exit 1 +fi + +fade_length=$1 +first_file=$2 +second_file=$3 + +fade_first="auto" +if [ "$4" != "" ]; then + fade_first=$4 +fi + +fade_second="auto" +if [ "$5" != "" ]; then + fade_second=$5 +fi + +fade_first_opts= +if [ "$fade_first" != "no" ]; then + fade_first_opts="fade t 0 0:0:$fade_length 0:0:$fade_length" +fi + +fade_second_opts= +if [ "$fade_second" != "no" ]; then + fade_second_opts="fade t 0:0:$fade_length" +fi + +echo "crossfade and concatenate files" +echo +echo "Finding length of $first_file..." +first_length=`$SOX "$first_file" 2>&1 -n stat | grep Length | cut -d : -f 2 | cut -f 1` +echo "Length is $first_length seconds" + +trim_length=`echo "$first_length - $fade_length" | bc` + +# Get crossfade section from first file and optionally do the fade out +echo "Obtaining $fade_length seconds of fade out portion from $first_file..." +$SOX "$first_file" -e signed-integer -b 16 fadeout1.wav trim $trim_length + +# When user specifies "auto" try to guess if a fadeout is needed. +# "RMS amplitude" from the stat effect is effectively an average +# value of samples for the whole fade length file. If it seems +# quite then assume a fadeout has already been done. An RMS value +# of 0.1 was just obtained from trail and error. +if [ "$fade_first" == "auto" ]; then + RMS=`$SOX fadeout1.wav 2>&1 -n stat | grep RMS | grep amplitude | cut -d : -f 2 | cut -f 1` + should_fade=`echo "$RMS > 0.1" | bc` + if [ $should_fade == 0 ]; then + echo "Auto mode decided not to fadeout with RMS of $RMS" + fade_first_opts="" + else + echo "Auto mode will fadeout" + fi +fi + +$SOX fadeout1.wav fadeout2.wav $fade_first_opts + +# Get the crossfade section from the second file and optionally do the fade in +echo "Obtaining $fade_length seconds of fade in portion from $second_file..." +$SOX "$second_file" -e signed-integer -b 16 fadein1.wav trim 0 $fade_length + +# For auto, do similar thing as for fadeout. +if [ "$fade_second" == "auto" ]; then + RMS=`$SOX fadein1.wav 2>&1 -n stat | grep RMS | grep amplitude | cut -d : -f 2 | cut -f 1` + should_fade=`echo "$RMS > 0.1" | bc` + if [ $should_fade == 0 ]; then + echo "Auto mode decided not to fadein with RMS of $RMS" + fade_second_opts="" + else + echo "Auto mode will fadein" + fi +fi + +$SOX fadein1.wav fadein2.wav $fade_second_opts + +# Mix the crossfade files together at full volume +echo "Crossfading..." +$SOX -m -v 1.0 fadeout2.wav -v 1.0 fadein2.wav crossfade.wav + +echo "Trimming off crossfade sections from original files..." + +$SOX "$first_file" -e signed-integer -b 16 song1.wav trim 0 $trim_length +$SOX "$second_file" -e signed-integer -b 16 song2.wav trim $fade_length +$SOX song1.wav crossfade.wav song2.wav mix.wav + +echo -e "Removing temporary files...\n" +rm fadeout1.wav fadeout2.wav fadein1.wav fadein2.wav crossfade.wav song1.wav song2.wav +mins=`echo "$trim_length / 60" | bc` +secs=`echo "$trim_length % 60" | bc` +echo "The crossfade in mix.wav occurs at around $mins mins $secs secs" + diff --git a/bin/loop.sh b/bin/loop.sh new file mode 100755 index 0000000..beb1286 --- /dev/null +++ b/bin/loop.sh @@ -0,0 +1,20 @@ +rm fades.txt +rm files.txt +rm mix.sh + +ITER=0 +NEXT=1 +for i in `ls -v *.wav` +do +echo "[0$ITER][$NEXT]acrossfade=ns=5:o=1:c1=tri:c2=tri[0$NEXT];" >> fades.txt +printf '\-i %s ' $i >> files.txt +ITER=$(expr $ITER + 1) +NEXT=$(expr $NEXT + 1) +done + +printf "ffmpeg " >> mix.sh +cat files.txt >> mix.sh +printf " -filter_complex \"" >> mix.sh +cat fades.txt >> mix.sh +echo '\" out.wav' >> mix.sh + diff --git a/bin/xfade.sh b/bin/xfade.sh new file mode 100755 index 0000000..5e861e0 --- /dev/null +++ b/bin/xfade.sh @@ -0,0 +1,23 @@ +crossfade_dur=1 +i=0 +limit=10000 + +for file in `ls *.wav | sort -V` +do + i=$((i+1)) + if [ $i -eq $limit ] + then + break + fi + + if [ $i -eq 1 ] + then + cp $file mix.wav + else + # ../../crossfade_cat.sh $crossfade_dur mix.wav $file yes yes + echo $file + sox mix.wav "$file" out.wav splice $(soxi -D mix.wav),0.01 + mv out.wav mix.wav + fi +done + diff --git a/datasets/count_subdirs.sh b/datasets/count_subdirs.sh new file mode 100755 index 0000000..3999b3c --- /dev/null +++ b/datasets/count_subdirs.sh @@ -0,0 +1 @@ +find -maxdepth 1 -type d | sort | while read -r dir; do printf "%s:\t" "$dir"; find "$dir" -type f | wc -l; done diff --git a/datasets/dataset.pl b/datasets/dataset.pl new file mode 100755 index 0000000..75aa2b1 --- /dev/null +++ b/datasets/dataset.pl @@ -0,0 +1,75 @@ +#!/usr/bin/perl + +use strict; +use Getopt::Std; + +our $opt_c; +getopts('c'); + +my $fmt = <<FMT; +Input File : '01_snapping_clean.wav' +Channels : 1 +Sample Rate : 44100 +Precision : 16-bit +Duration : 00:03:09.35 = 8350438 samples = 14201.4 CDDA sectors +File Size : 16.7M +Bit Rate : 706k +Sample Encoding: 16-bit Signed Integer PCM +FMT + +sub process($) { + my $filename = shift or die "Usage: $0 [...filenames]\n"; + my ($name, $ext) = split(/\./, $filename, 2); + if ($ext !~ /(wav|mp3|aiff?|flac)/) { + print "not a valid file extension: $ext\n"; + return; + } + + if ($ext eq 'mp3') { + system('ffmpeg', '-i', $filename, $name . '.wav'); + $filename = $name . '.wav'; + } + my $soxi = `soxi $filename`; + my @lines = split("\n", $soxi); + + print $soxi; + + my $seconds; + for my $line (@lines) { + if ($line =~ /Duration : (\d\d):(\d\d):(\d\d)\./) { + my $h = $1; + my $m = $2; + my $s = $3; + $seconds = (((($h * 60) + $m) * 60) + $s) + 0; + } + } + + my $scale = sprintf("%.09f", 5e-7 * $seconds); + + print "Seconds: $seconds\n"; + print "Scale factor: $scale\n"; + print "\n"; + + my $a_tmp = "a_" . $filename; + my $b_tmp = "b_" . $filename; + if (!$opt_c) { + print "Normalizing..."; + system("sox", "-v", 0.945, $filename, $a_tmp); + } else { + $a_tmp = $filename; + } + system("./spread.sh", $a_tmp, $b_tmp, 0.999, $scale, 1.001); + system("./split44k.sh", $b_tmp, 8, $name); + if (!$opt_c) { + system("/bin/rm", $a_tmp); + } + system("/bin/rm", $b_tmp); +} + +foreach my $file (@ARGV) {; + if ( -e $file ) { + process($file); + } +} + + diff --git a/datasets/generate.sh b/datasets/generate.sh new file mode 100755 index 0000000..335928c --- /dev/null +++ b/datasets/generate.sh @@ -0,0 +1,34 @@ +function process () { + echo "____________________________________________________" + echo "process $1" + name=$1 + in="${name}.wav" + out="s_${in}" + ./spread.sh $in $out 0.99 0.01 1.01 + ./split44k.sh $out 8 "44k_$name" + rm $out +} +function ease_process () { + echo "____________________________________________________" + echo "ease_process $1" + name=$1 + step=$2 + in="${name}.wav" + sout="o_${in}" + out="s_${in}" + sox -v 0.95 $in $sout + ./spread.sh $sout $out 0.999 $step 1.001 + ./split44k.sh $out 8 "44k_$name" + rm $sout + rm $out +} +#ease_process '' 0.0000 +ease_process 'blblbl' 0.00001515 +ease_process 'faty-scrub1' 0.0000285 +ease_process 'faty-medieval' 0.00003 +ease_process 'faty-crystals' 0.0000111 +ease_process 'faty-vocal1' 0.000013 +ease_process 'faty-vocal2' 0.000028145 +ease_process 'faty-scrub2' 0.00000466 +ease_process 'siren' 0.0000275 + diff --git a/datasets/get.pl b/datasets/get.pl new file mode 100755 index 0000000..14c5a5b --- /dev/null +++ b/datasets/get.pl @@ -0,0 +1,9 @@ +#!/usr/bin/perl + +foreach (@ARGV){ + my $s = $_; + if ($s !~/\..*$/) { $s .= ".wav"; } + system('/usr/bin/wget', 'https://neural:spawn5@asdf.us/neural/' . $s); + system('/usr/bin/perl', 'dataset.pl', $s); +} + diff --git a/datasets/silence.sh b/datasets/silence.sh new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/datasets/silence.sh diff --git a/datasets/split44k.sh b/datasets/split44k.sh new file mode 100755 index 0000000..b679d29 --- /dev/null +++ b/datasets/split44k.sh @@ -0,0 +1,24 @@ +#/bin/sh + +if [ "$#" -ne 3 ]; then + echo "Usage: $0 <filename.wav> <chunk size in seconds> <dataset path>" + exit +fi + +fn=$1 +chunk_size=$2 +dataset_path=$3 + +converted=".temp2.wav" +rm -f $converted +ffmpeg -hide_banner -loglevel error -i $fn -ac 1 -ar 44100 $converted + +mkdir $dataset_path +length=$(ffprobe -i $converted -show_entries format=duration -v quiet -of csv="p=0") +end=$(echo "$length / $chunk_size - 1" | bc) +echo "splitting..." +for i in $(seq 0 $end); do + ffmpeg -hide_banner -loglevel error -ss $(($i * $chunk_size)) -t $chunk_size -i $converted "$dataset_path/$i.wav" +done +echo "done" +rm -f $converted diff --git a/datasets/spread.sh b/datasets/spread.sh new file mode 100755 index 0000000..60c3551 --- /dev/null +++ b/datasets/spread.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +if [ "$#" -ne 5 ]; then + echo "Usage: $0 <in.wav> <out.wav> <rate_min> <rate_step> <rate_max>" + exit +fi + +FN_IN=$1 +FN_OUT=$2 +RATE=$3 +STEP=$4 +MAX=$5 + +ITER=0 + +while true; do + if (( $(echo "$RATE > $MAX" | bc -l) )); then + break + fi + let ITER+=1 + RATE=`echo "$RATE+$STEP" | bc` + if ((ITER % 20 != 0)); then + echo "${ITER}... ${RATE}" + sleep 1 + fi + sox $FN_IN "tmp_$ITER.wav" speed $RATE +done + +echo "made $ITER copies" + +sox tmp_* $FN_OUT +rm tmp_* + diff --git a/epoch.sh b/epoch.sh new file mode 100755 index 0000000..bbd8668 --- /dev/null +++ b/epoch.sh @@ -0,0 +1 @@ +ls -lat results/*/checkpoints/ep* | head -n 20 diff --git a/export-all.pl b/export-all.pl new file mode 100644 index 0000000..e840729 --- /dev/null +++ b/export-all.pl @@ -0,0 +1,28 @@ +#!/usr/bin/perl + +use strict; + +sub process($){ + my ($dir) = @_; + my @exp = split(":", $dir); + my $name = $exp[4]; + my $ep; + my $path = "results/" . $dir . "/"; + my $chex = $path . "checkpoints/"; + return 0 unless -e $chex; + my $sampz = $path . "samples/"; + return 0 unless -e $sampz; + + # print $name . "\n"; + system('/bin/bash', 'mix-wav.sh', $path, $name); +} + +opendir RESULTS, ("results/") or die $!; +# my @results = sort {(stat $a)[9] <=> (stat $b)[9]} readdir(RESULTS); +my @results = sort {$a cmp $b} readdir(RESULTS); +closedir RESULTS; + +for my $result (@results) { + next if $result !~ /exp:/; + process($result); +} diff --git a/generate.py b/generate.py new file mode 100644 index 0000000..92a930f --- /dev/null +++ b/generate.py @@ -0,0 +1,365 @@ +# CometML needs to be imported first. +try: + import comet_ml +except ImportError: + pass + +from model import SampleRNN, Predictor +from optim import gradient_clipping +from nn import sequence_nll_loss_bits +from trainer import Trainer +from trainer.plugins import ( + TrainingLossMonitor, ValidationPlugin, AbsoluteTimeMonitor, SaverPlugin, + GeneratorPlugin, StatsPlugin +) +from dataset import FolderDataset, DataLoader + +import torch +from torch.utils.trainer.plugins import Logger + +from natsort import natsorted + +from functools import reduce +import os +import shutil +import sys +from glob import glob +import re +import argparse + + +default_params = { + # model parameters + 'n_rnn': 1, + 'dim': 1024, + 'learn_h0': True, + 'q_levels': 256, + 'seq_len': 1024, + 'weight_norm': True, + 'batch_size': 128, + 'val_frac': 0.1, + 'test_frac': 0.1, + + # training parameters + 'keep_old_checkpoints': False, + 'datasets_path': 'datasets', + 'results_path': 'results', + 'epoch_limit': 1000, + 'resume': True, + 'sample_rate': 16000, + 'n_samples': 1, + 'sample_length': 80000, + 'loss_smoothing': 0.99, + 'cuda': True, + 'comet_key': None +} + +tag_params = [ + 'exp', 'frame_sizes', 'n_rnn', 'dim', 'learn_h0', 'q_levels', 'seq_len', + 'batch_size', 'dataset', 'val_frac', 'test_frac' +] + +def param_to_string(value): + if isinstance(value, bool): + return 'T' if value else 'F' + elif isinstance(value, list): + return ','.join(map(param_to_string, value)) + else: + return str(value) + +def make_tag(params): + return '-'.join( + key + ':' + param_to_string(params[key]) + for key in tag_params + if key not in default_params or params[key] != default_params[key] + ) + +def setup_results_dir(params): + def ensure_dir_exists(path): + if not os.path.exists(path): + os.makedirs(path) + + tag = make_tag(params) + results_path = os.path.abspath(params['results_path']) + ensure_dir_exists(results_path) + results_path = os.path.join(results_path, tag) + if not os.path.exists(results_path): + os.makedirs(results_path) + elif not params['resume']: + shutil.rmtree(results_path) + os.makedirs(results_path) + + for subdir in ['checkpoints', 'samples']: + ensure_dir_exists(os.path.join(results_path, subdir)) + + return results_path + +def load_last_checkpoint(checkpoints_path): + checkpoints_pattern = os.path.join( + checkpoints_path, SaverPlugin.last_pattern.format('*', '*') + ) + checkpoint_paths = natsorted(glob(checkpoints_pattern)) + if len(checkpoint_paths) > 0: + checkpoint_path = checkpoint_paths[-1] + checkpoint_name = os.path.basename(checkpoint_path) + match = re.match( + SaverPlugin.last_pattern.format(r'(\d+)', r'(\d+)'), + checkpoint_name + ) + epoch = int(match.group(1)) + iteration = int(match.group(2)) + return (torch.load(checkpoint_path), epoch, iteration) + else: + return None + +def tee_stdout(log_path): + log_file = open(log_path, 'a', 1) + stdout = sys.stdout + + class Tee: + + def write(self, string): + log_file.write(string) + stdout.write(string) + + def flush(self): + log_file.flush() + stdout.flush() + + sys.stdout = Tee() + +def make_data_loader(overlap_len, params): + path = os.path.join(params['datasets_path'], params['dataset']) + def data_loader(split_from, split_to, eval): + dataset = FolderDataset( + path, overlap_len, params['q_levels'], split_from, split_to + ) + return DataLoader( + dataset, + batch_size=params['batch_size'], + seq_len=params['seq_len'], + overlap_len=overlap_len, + shuffle=(not eval), + drop_last=(not eval) + ) + return data_loader + +def init_comet(params, trainer): + if params['comet_key'] is not None: + from comet_ml import Experiment + from trainer.plugins import CometPlugin + experiment = Experiment(api_key=params['comet_key'], log_code=False) + hyperparams = { + name: param_to_string(params[name]) for name in tag_params + } + experiment.log_multiple_params(hyperparams) + trainer.register_plugin(CometPlugin( + experiment, [ + ('training_loss', 'epoch_mean'), + 'validation_loss', + 'test_loss' + ] + )) + +def main(exp, frame_sizes, dataset, **params): + params = dict( + default_params, + exp=exp, frame_sizes=frame_sizes, dataset=dataset, + **params + ) + + results_path = setup_results_dir(params) + tee_stdout(os.path.join(results_path, 'log')) + + model = SampleRNN( + frame_sizes=params['frame_sizes'], + n_rnn=params['n_rnn'], + dim=params['dim'], + learn_h0=params['learn_h0'], + q_levels=params['q_levels'], + weight_norm=params['weight_norm'] + ) + predictor = Predictor(model) + if params['cuda']: + model = model.cuda() + predictor = predictor.cuda() + + optimizer = gradient_clipping(torch.optim.Adam(predictor.parameters())) + + data_loader = make_data_loader(model.lookback, params) + test_split = 1 - params['test_frac'] + val_split = test_split - params['val_frac'] + + trainer = Trainer( + predictor, sequence_nll_loss_bits, optimizer, + data_loader(0, val_split, eval=False), + cuda=params['cuda'] + ) + + checkpoints_path = os.path.join(results_path, 'checkpoints') + checkpoint_data = load_last_checkpoint(checkpoints_path) + if checkpoint_data is not None: + (state_dict, epoch, iteration) = checkpoint_data + trainer.epochs = epoch + trainer.iterations = iteration + predictor.load_state_dict(state_dict) + print("epochs: {} iterations: {}".format(epoch, iteration)) + + """ + trainer.register_plugin(TrainingLossMonitor( + smoothing=params['loss_smoothing'] + )) + """ + trainer.register_plugin(ValidationPlugin( + data_loader(val_split, test_split, eval=True), + data_loader(test_split, 1, eval=True) + )) + trainer.register_plugin(AbsoluteTimeMonitor()) + """ + trainer.register_plugin(SaverPlugin( + checkpoints_path, params['keep_old_checkpoints'] + )) + """ + trainer.register_plugin(GeneratorPlugin( + os.path.join(results_path, 'samples'), params['n_samples'], + params['sample_length'], params['sample_rate'] + )) + """ + trainer.register_plugin( + Logger([ + 'training_loss', + 'validation_loss', + 'test_loss', + 'time' + ]) + ) + trainer.register_plugin(StatsPlugin( + results_path, + iteration_fields=[ + 'training_loss', + ('training_loss', 'running_avg'), + 'time' + ], + epoch_fields=[ + 'validation_loss', + 'test_loss', + 'time' + ], + plots={ + 'loss': { + 'x': 'iteration', + 'ys': [ + 'training_loss', + ('training_loss', 'running_avg'), + 'validation_loss', + 'test_loss', + ], + 'log_y': True + } + } + )) + init_comet(params, trainer) + """ + trainer.generate(int(params['epoch_limit'])) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + argument_default=argparse.SUPPRESS + ) + + def parse_bool(arg): + arg = arg.lower() + if 'true'.startswith(arg): + return True + elif 'false'.startswith(arg): + return False + else: + raise ValueError() + + parser.add_argument('--exp', required=True, help='experiment name') + parser.add_argument( + '--frame_sizes', nargs='+', type=int, required=True, + help='frame sizes in terms of the number of lower tier frames, \ + starting from the lowest RNN tier' + ) + parser.add_argument( + '--dataset', required=True, + help='dataset name - name of a directory in the datasets path \ + (settable by --datasets_path)' + ) + parser.add_argument( + '--n_rnn', type=int, help='number of RNN layers in each tier' + ) + parser.add_argument( + '--dim', type=int, help='number of neurons in every RNN and MLP layer' + ) + parser.add_argument( + '--learn_h0', type=parse_bool, + help='whether to learn the initial states of RNNs' + ) + parser.add_argument( + '--q_levels', type=int, + help='number of bins in quantization of audio samples' + ) + parser.add_argument( + '--seq_len', type=int, + help='how many samples to include in each truncated BPTT pass' + ) + parser.add_argument( + '--weight_norm', type=parse_bool, + help='whether to use weight normalization' + ) + parser.add_argument('--batch_size', type=int, help='batch size') + parser.add_argument( + '--val_frac', type=float, + help='fraction of data to go into the validation set' + ) + parser.add_argument( + '--test_frac', type=float, + help='fraction of data to go into the test set' + ) + parser.add_argument( + '--keep_old_checkpoints', type=parse_bool, + help='whether to keep checkpoints from past epochs' + ) + parser.add_argument( + '--datasets_path', help='path to the directory containing datasets' + ) + parser.add_argument( + '--results_path', help='path to the directory to save the results to' + ) + parser.add_argument('--epoch_limit', help='how many epochs to run') + parser.add_argument( + '--resume', type=parse_bool, default=True, + help='whether to resume training from the last checkpoint' + ) + parser.add_argument( + '--sample_rate', type=int, + help='sample rate of the training data and generated sound' + ) + parser.add_argument( + '--n_samples', type=int, + help='number of samples to generate in each epoch' + ) + parser.add_argument( + '--sample_length', type=int, + help='length of each generated sample (in samples)' + ) + parser.add_argument( + '--loss_smoothing', type=float, + help='smoothing parameter of the exponential moving average over \ + training loss, used in the log and in the loss plot' + ) + parser.add_argument( + '--cuda', type=parse_bool, + help='whether to use CUDA' + ) + parser.add_argument( + '--comet_key', help='comet.ml API key' + ) + + parser.set_defaults(**default_params) + + main(**vars(parser.parse_args())) @@ -0,0 +1,57 @@ +#!/usr/bin/perl + +$SIG{TERM} = $SIG{INT} = sub { exit 1 }; + +chdir('datasets'); + +my $fn, $new_fn; +foreach my $s (@ARGV){ + if ($s =~ /^http/) { + if ($s =~ /(wav|aiff?|flac|mp3|opus)$/i) { + my $fn = `basename $s`; + print "downloading $fn\n"; + system('/usr/bin/wget', $s); + system('/usr/bin/perl', 'dataset.pl', $fn); + } else { + print "youtube-dl $s\n"; + my $yt = `youtube-dl --extract-audio --audio-format flac -o "%(title)s.%(ext)s" $s`; + my @partz = split("\n", $yt); + foreach $part (@partz) { + if ($part =~ /\[ffmpeg\] Destination\: (.*\.flac)$/) { + $fn = $1; + } + } + if ($fn) { + $new_fn = lc $fn; + $new_fn =~ s/\.flac$//g; + $new_fn =~ s/\s+/_/g; + $new_fn =~ s/\W//g; + if (length($new_fn) == 0) { + $new_fn = lc $s; + $new_fn =~ s/\s+/_/g; + $new_fn =~ s/\W//g; + } + $new_fn =~ s/_+/_/g; + $new_fn .= '.flac'; + system('mv', $fn, $new_fn); + print"got fn, $fn => $new_fn\n"; + system('/usr/bin/perl', 'dataset.pl', $new_fn); + } + } + } else { + if ($s !~/\..*$/) { $s .= ".wav"; } + print "downloading $s\n"; + system('/usr/bin/wget', 'https://neural:spawn5@asdf.us/neural/' . $s); + system('/usr/bin/perl', 'dataset.pl', $s); + } + open(my $fd, ">>../run_slap.sh"); + print $fd "standard $fn"; + close $fn; + + $fn = undef; +} + +END { + chdir('..'); +} + diff --git a/glass_test2.sh b/glass_test2.sh new file mode 100755 index 0000000..072c451 --- /dev/null +++ b/glass_test2.sh @@ -0,0 +1,85 @@ +function runq () { + dataset=$1 + sample_rate=$2 + duration=$3 + let sample_length=$2*$3 + fs1=$4 + fs2=$5 + qs=$6 + + exp_name="space_q$qs" + + echo "__________________________________________" + echo "__________________________________________" + echo "__________________________________________" + echo "" + echo ">> running $exp_name" + echo "" + echo "__________________________________________" + echo "__________________________________________" + echo "__________________________________________" + + python train.py \ + --exp $exp_name --dataset $dataset \ + --frame_sizes $fs1 $fs2 \ + --n_rnn 2 --dim 1024 --q_levels 512 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit 6 \ + --resume True +} +function generateq () { + dataset=$1 + sample_rate=$2 + duration=$3 + let sample_length=$2*$3 + fs1=$4 + fs2=$5 + qs=$6 + + exp_name="space_q$qs" + + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo "" + echo ">> generating $exp_name" + echo "" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + + python generate.py \ + --exp $exp_name --dataset $dataset \ + --frame_sizes $fs1 $fs2 \ + --n_rnn 2 --dim 1024 --q_levels 512 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit 6 \ + --resume True +} + +#runq 44k_glass_space1 44100 5 8 2 32 +runq 44k_glass_space1 44100 5 8 2 1 +runq 44k_glass_space1 44100 5 8 2 2 +runq 44k_glass_space1 44100 5 8 2 4 +runq 44k_glass_space1 44100 5 8 2 8 +runq 44k_glass_space1 44100 5 8 2 16 +runq 44k_glass_space1 44100 5 8 2 32 +#runq 44k_glass_space1 44100 5 8 2 2048 + +#generateq 44k_glass_space1 44100 5 8 2 32 +#generateq 44k_glass_space1 44100 5 8 2 1 +#generateq 44k_glass_space1 44100 5 8 2 2 +#generateq 44k_glass_space1 44100 5 8 2 4 +#generateq 44k_glass_space1 44100 5 8 2 8 +#generateq 44k_glass_space1 44100 5 8 2 16 +#generateq 44k_glass_space1 44100 5 8 2 32 +#generateq 44k_glass_space1 44100 5 8 2 2048 + diff --git a/glass_test3.sh b/glass_test3.sh new file mode 100755 index 0000000..029cf58 --- /dev/null +++ b/glass_test3.sh @@ -0,0 +1,112 @@ +function runq () { + dataset=$1 + sample_rate=$2 + duration=$3 + let sample_length=$2*$3 + qs=$4 + + exp_name="space_qs$dim" + + echo "__________________________________________" + echo "__________________________________________" + echo "__________________________________________" + echo "" + echo ">> running $exp_name" + echo "" + echo "__________________________________________" + echo "__________________________________________" + echo "__________________________________________" + + python train.py \ + --exp $exp_name --dataset $dataset \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels $qs \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit 4 \ + --resume True +} + +function rundim () { + dataset=$1 + sample_rate=$2 + duration=$3 + let sample_length=$2*$3 + dim=$4 + + exp_name="space_dim$dim" + + echo "__________________________________________" + echo "__________________________________________" + echo "__________________________________________" + echo "" + echo ">> running $exp_name" + echo "" + echo "__________________________________________" + echo "__________________________________________" + echo "__________________________________________" + + python train.py \ + --exp $exp_name --dataset $dataset \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim $dim --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit 8 \ + --resume True +} + +function runseq () { + dataset=$1 + sample_rate=$2 + duration=$3 + let sample_length=$2*$3 + seq=$4 + + exp_name="space_seq$seq" + + echo "__________________________________________" + echo "__________________________________________" + echo "__________________________________________" + echo "" + echo ">> running $exp_name" + echo "" + echo "__________________________________________" + echo "__________________________________________" + echo "__________________________________________" + + python train.py \ + --exp $exp_name --dataset $dataset \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len $seq --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit 4 \ + --resume True +} + +#rundim 44k_glass_space1 44100 5 4096 +runseq 44k_glass_space1 44100 5 512 +rundim 44k_glass_space1 44100 5 512 +#rundim 44k_glass_space1 44100 5 256 +#rundim 44k_glass_space1 44100 5 128 +#rundim 44k_glass_space1 44100 5 64 +#rundim 44k_glass_space1 44100 5 32 +#rundim 44k_glass_space1 44100 5 16 +#rundim 44k_glass_space1 44100 5 8 +#rundim 44k_glass_space1 44100 5 4 +#rundim 44k_glass_space1 44100 5 2 +#rundim 44k_glass_space1 44100 5 1 + +runq 44k_glass_space1 44100 5 1024 +runq 44k_glass_space1 44100 5 128 + diff --git a/glass_tests.sh b/glass_tests.sh new file mode 100755 index 0000000..62d8932 --- /dev/null +++ b/glass_tests.sh @@ -0,0 +1,202 @@ +function run2 () { + dataset=$1 + sample_rate=$2 + duration=$3 + let sample_length=$2*$3 + fs1=$4 + fs2=$5 + + exp_name=space_2 + + echo "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@" + echo "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@" + echo "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@" + echo "" + echo ">> running $exp_name $4 $5" + echo "" + echo "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@" + echo "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@" + echo "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@" + python train.py \ + --exp $exp_name --dataset $dataset \ + --frame_sizes $fs1 $fs2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit 2 \ + --resume True +} +function generate2 () { + dataset=$1 + sample_rate=$2 + duration=$3 + let sample_length=$2*$3 + fs1=$4 + fs2=$5 + + exp_name=space_2 + + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo "" + echo ">> generating $exp_name $4 $5" + echo "" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + python generate.py \ + --exp $exp_name --dataset $dataset \ + --frame_sizes $fs1 $fs2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit 1 \ + --n_samples 6 \ + --resume True +} +function run3 () { + dataset=$1 + sample_rate=$2 + duration=$3 + let sample_length=$2*$3 + fs1=$4 + fs2=$5 + fs3=$6 + + exp_name=space_3 + + echo ">> running $exp_name $4 $5 $6" + python train.py \ + --exp $exp_name --dataset $dataset \ + --frame_sizes $fs1 $fs2 $fs3 \ + --n_rnn 3 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit 1 \ + --resume True +} + +function run4 () { + dataset=$1 + sample_rate=$2 + duration=$3 + let sample_length=$2*$3 + fs1=$4 + fs2=$5 + fs3=$6 + fs4=$7 + + exp_name=space_4 + + echo ">> running $exp_name $4 $5 $6 $7" + python train.py \ + --exp $exp_name --dataset $dataset \ + --frame_sizes $fs1 $fs2 $fs3 $fs4 \ + --n_rnn 4 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit 4 \ + --resume True +} +function run5 () { + dataset=$1 + sample_rate=$2 + duration=$3 + let sample_length=$2*$3 + fs1=$4 + fs2=$5 + fs3=$6 + fs4=$7 + fs5=$8 + + exp_name=space_5 + + echo ">> running $exp_name $4 $5 $6 $7 $8" + python train.py \ + --exp $exp_name --dataset $dataset \ + --frame_sizes $fs1 $fs2 $fs3 $fs4 $fs5 \ + --n_rnn 5 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit 4 \ + --resume True +} +function run6 () { + dataset=$1 + sample_rate=$2 + duration=$3 + let sample_length=$2*$3 + fs1=$4 + fs2=$5 + fs3=$6 + fs4=$7 + fs5=$8 + fs6=$9 + + exp_name=space_6 + + echo ">> running $exp_name $4 $5 $6 $7 $8 $9" + python train.py \ + --exp $exp_name --dataset $dataset \ + --frame_sizes $fs1 $fs2 $fs3 $fs4 $fs5 $fs6 \ + --n_rnn 5 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit 4 \ + --resume True +} + +run4 44k_glass_space1 44100 5 2 2 2 2 +run5 44k_glass_space1 44100 5 2 2 2 2 2 +run6 44k_glass_space1 44100 5 2 2 2 2 2 2 + +run4 44k_glass_space1 44100 5 8 2 2 2 +run5 44k_glass_space1 44100 5 8 2 2 2 2 +run6 44k_glass_space1 44100 5 8 2 2 2 2 2 +#generate2 44k_glass_space1 44100 5 8 2 +#run2 44k_glass_space1 44100 5 8 4 +#generate2 44k_glass_space1 44100 5 8 4 +#run2 44k_glass_space1 44100 5 8 1 +#generate2 44k_glass_space1 44100 5 8 1 +#run2 44k_glass_space1 44100 5 16 8 +#run2 44k_glass_space1 44100 5 16 4 +#run2 44k_glass_space1 44100 5 16 2 +#generate2 44k_glass_space1 44100 5 16 2 +#run2 44k_glass_space1 44100 5 16 1 +#generate2 44k_glass_space1 44100 5 16 1 + +#run2 44k_glass_space1 44100 5 4 2 +#run2 44k_glass_space1 44100 5 4 1 +#run2 44k_glass_space1 44100 5 2 1 + +#run3 44k_glass_space1 44100 5 8 2 1 +#run3 44k_glass_space1 44100 5 8 4 2 +#run3 44k_glass_space1 44100 5 16 8 2 +#run3 44k_glass_space1 44100 5 16 4 2 +#run3 44k_glass_space1 44100 5 16 2 1 + +#run2 44k_glass_space1 44100 5 6 2 +#run2 44k_glass_space1 44100 5 6 3 +#run3 44k_glass_space1 44100 5 5 3 2 +#run3 44k_glass_space1 44100 5 10 5 2 +#run3 44k_glass_space1 44100 5 8 5 2 + diff --git a/kick_test.sh b/kick_test.sh new file mode 100755 index 0000000..c3754a4 --- /dev/null +++ b/kick_test.sh @@ -0,0 +1,51 @@ +function generaterrr () { + exp_name=$1 + dataset=$2 + epoch_limit=$3 + sample_rate=$4 + duration=$5 + let sample_length=$4*$5 + qlev=$6 + + echo ">> generating $exp_name" + python generate.py \ + --exp $exp_name --dataset $dataset \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels $qlev \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit $epoch_limit \ + --resume True +} +function runrrr () { + exp_name=$1 + dataset=$2 + epoch_limit=$3 + sample_rate=$4 + duration=$5 + let sample_length=$4*$5 + qlev=$6 + + echo ">> running $exp_name" + python train.py \ + --exp $exp_name --dataset $dataset \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels $qlev \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit $epoch_limit \ + --resume True +} + +runrrr kiq256 kiq 1 44100 1 256 +generaterrr kiq256 kiq 6 44100 1 256 + +runrrr kiq512 kiq 1 44100 1 512 +generaterrr kiq512 kiq 10 44100 1 512 + diff --git a/latest.pl b/latest.pl new file mode 100755 index 0000000..1bb7932 --- /dev/null +++ b/latest.pl @@ -0,0 +1,106 @@ +#!/usr/bin/perl + +use strict; +use Getopt::Std; + +our $opt_l; # mix and upload files +our $opt_v; # print log for all files +our $opt_n; # name/tag for file +getopts('lavn:'); + +our $matches = 0; +if (scalar @ARGV) { + $matches = {}; + for (@ARGV) { + my ($name, $ep) = split("-", $_, 2); + $matches->{$name} = 1; + } +} +# my $s = "results/exp:03_responses-frame_sizes:8,2-n_rnn:2-dataset:03_responses/checkpoints/best-ep1-it1035"; +sub process($){ + my ($dir) = @_; + my @exp = split(":", $dir); + my $name = $exp[4]; + my $ep; + my $path = "results/" . $dir . "/"; + my $chex = $path . "checkpoints/"; + my $log = $path . 'log'; + return 0 unless -e $chex; + opendir DIR, $chex or die $!; + while (readdir DIR) { + next unless /^ep/; + $ep = $_; + last; + } + closedir DIR; + + my $sampz = $path . "samples/"; + return 0 unless -e $sampz; + my $c = 0; + opendir DIR, $sampz or die $!; + while (readdir DIR) { + next unless /wav$/; + $c += 1; + } + closedir DIR; + + if ($matches && ! exists $matches->{$name}) { + return 0; + } + + if ($opt_v) { + print $name . "\n"; + print `grep valid $log`; + print "\n"; + } + + if ($c == 0) { + return 0; + } + + my @epoch = split("-", $ep); + $name .= "-" . $epoch[0]; + + if (! $epoch[0]) { + return 0; + } + + if (! $opt_l) { + print $name . ", $c samples\n"; + } + + $epoch[0] =~ /(\d+)/; + #if ($1 < 4) { + # return 0; + #} + + if ($opt_l && $opt_n) { + $name .= '-' . $opt_n; + } + + if (-e "output/" . $name . ".mp3") { + return 1; + } + + if (!$opt_l) { + return 0; + } + print "\n"; + print "_______________________________________________________________\n"; + print "\n"; + print $name . ", $c samples\n"; + print "\n"; + system('/bin/bash', 'mix.sh', $path, $name); + return 0; +} + +opendir RESULTS, ("results/") or die $!; +# my @results = sort {(stat $a)[9] <=> (stat $b)[9]} readdir(RESULTS); +my @results = sort {$a cmp $b} readdir(RESULTS); +closedir RESULTS; + +for my $result (@results) { + next if $result !~ /exp:/; + my $rv = process($result); + #last if $rv; +} diff --git a/mix-wav.sh b/mix-wav.sh new file mode 100644 index 0000000..5348309 --- /dev/null +++ b/mix-wav.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 <results/exp\:experiment> render_name" + exit +fi + +dir=$1 +name=$2 +now=`date +'%Y%m%d'` + +# echo "rendering $name" + +cd "$dir/samples" +for i in * +do + if [[ -d $i ]]; then + cd $i + echo $i + ../../../xfade.sh + sox mix.wav norm.wav norm + mv norm.wav "../../../../wav/${i}.wav" + cd .. + fi +done +cd ../../.. @@ -0,0 +1,25 @@ +#!/bin/bash + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 <results/exp\:experiment> render_name" + exit +fi + +dir=$1 +name=$2 +now=`date +'%Y%m%d'` + +echo "rendering $name" + +cd "$dir/samples" +mkdir "$name" +../../xfade.sh +lame -V 0 mix.wav "../../../output/$name.mp3" +rm mix.wav +mv *.wav "$name" +cd .. +grep valid log +cd ../.. + +node upload.js "./output/$name.mp3" + diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..f53f640 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,995 @@ +{ + "name": "samplernn", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "@sindresorhus/is": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.7.0.tgz", + "integrity": "sha512-ONhaKPIufzzrlNbqtWFFd+jlnemX6lJAgq9ZeiZtS7I1PIf/la7CW4m83rTXRnVnsMbW2k56pGYu7AUFJD9Pow==" + }, + "@slack/client": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@slack/client/-/client-4.2.0.tgz", + "integrity": "sha512-FASaHhy8PPEN/h/eMy9edzwhaRRGfqZA8LklkBcz8z4VYrbWSYljoeup8CfFrl69Atbv4ILrRAPGefytQzoSUA==", + "requires": { + "@types/delay": "2.0.1", + "@types/form-data": "2.2.1", + "@types/got": "7.1.8", + "@types/is-stream": "1.1.0", + "@types/loglevel": "1.5.3", + "@types/node": "9.6.8", + "@types/p-cancelable": "0.3.0", + "@types/p-queue": "2.3.1", + "@types/p-retry": "1.0.1", + "@types/retry": "0.10.2", + "@types/url-join": "0.8.2", + "@types/ws": "4.0.2", + "delay": "2.0.0", + "eventemitter3": "3.1.0", + "finity": "0.5.4", + "form-data": "2.3.2", + "got": "8.3.1", + "is-stream": "1.1.0", + "loglevel": "1.6.1", + "object.entries": "1.0.4", + "object.getownpropertydescriptors": "2.0.3", + "object.values": "1.0.4", + "p-cancelable": "0.3.0", + "p-queue": "2.4.2", + "p-retry": "1.0.0", + "retry": "0.10.1", + "url-join": "4.0.0", + "ws": "4.1.0" + } + }, + "@types/delay": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@types/delay/-/delay-2.0.1.tgz", + "integrity": "sha512-D1/YuYOcdOIdaQnaiUJ77VcilVvESkynw79CtGqpjkXyv4OUezEVZtdXnSOwXL8Zcelu66QbyC8QQcVQ/ZPdig==" + }, + "@types/events": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@types/events/-/events-1.2.0.tgz", + "integrity": "sha512-KEIlhXnIutzKwRbQkGWb/I4HFqBuUykAdHgDED6xqwXJfONCjF5VoE0cXEiurh3XauygxzeDzgtXUqvLkxFzzA==" + }, + "@types/form-data": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/@types/form-data/-/form-data-2.2.1.tgz", + "integrity": "sha512-JAMFhOaHIciYVh8fb5/83nmuO/AHwmto+Hq7a9y8FzLDcC1KCU344XDOMEmahnrTFlHjgh4L0WJFczNIX2GxnQ==", + "requires": { + "@types/node": "9.6.8" + } + }, + "@types/got": { + "version": "7.1.8", + "resolved": "https://registry.npmjs.org/@types/got/-/got-7.1.8.tgz", + "integrity": "sha512-QxcSkx9PjHH7uqbzqKTKOAjGbayyo+dECnnqM3BBMC0WjYAqx0e6Qi9AFR4jluvx91e7qzgz4aGka7AhCTHYRw==", + "requires": { + "@types/node": "9.6.8" + } + }, + "@types/is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@types/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha512-jkZatu4QVbR60mpIzjINmtS1ZF4a/FqdTUTBeQDVOQ2PYyidtwFKr0B5G6ERukKwliq+7mIXvxyppwzG5EgRYg==", + "requires": { + "@types/node": "9.6.8" + } + }, + "@types/loglevel": { + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/@types/loglevel/-/loglevel-1.5.3.tgz", + "integrity": "sha512-TzzIZihV+y9kxSg5xJMkyIkaoGkXi50isZTtGHObNHRqAAwjGNjSCNPI7AUAv0tZUKTq9f2cdkCUd/2JVZUTrA==" + }, + "@types/node": { + "version": "9.6.8", + "resolved": "https://registry.npmjs.org/@types/node/-/node-9.6.8.tgz", + "integrity": "sha512-0PmgMBskTJa7zDyENW9C7Lunk+I0L2CHYF2RrBRljCmLSMM1fBHIIdvE1IboNNz7N6t+raJIj90YMvUYl2VT1g==" + }, + "@types/p-cancelable": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@types/p-cancelable/-/p-cancelable-0.3.0.tgz", + "integrity": "sha512-sP+9Ivnpil7cdmvr5O+145aXm65YX8Y+Lrul1ojdYz6yaE05Dqonn6Z9v5eqJCQ0UeSGcTRtepMlZDh9ywdKgw==" + }, + "@types/p-queue": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@types/p-queue/-/p-queue-2.3.1.tgz", + "integrity": "sha512-JyO7uMAtkcMMULmsTQ4t/lCC8nxirTtweGG1xAFNNIAoC1RemmeIxq8PiKghuEy99XdbS6Lwx4zpbXUjfeSSAA==" + }, + "@types/p-retry": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@types/p-retry/-/p-retry-1.0.1.tgz", + "integrity": "sha512-HgQPG9kkUb4EpTeUv2taH2nBZsVUb5aOTSw3X2YozcTG1ttmGcLaLKx1MbAz1evVfUEDTCAPmdz2HiFztIyWrw==", + "requires": { + "@types/retry": "0.10.2" + } + }, + "@types/retry": { + "version": "0.10.2", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.10.2.tgz", + "integrity": "sha512-LqJkY4VQ7S09XhI7kA3ON71AxauROhSv74639VsNXC9ish4IWHnIi98if+nP1MxQV3RMPqXSCYgpPsDHjlg9UQ==" + }, + "@types/url-join": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/@types/url-join/-/url-join-0.8.2.tgz", + "integrity": "sha1-EYHsvh2XtwNODqHjXmLobMJrQi0=" + }, + "@types/ws": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-4.0.2.tgz", + "integrity": "sha512-tlDVFHCcJdNqYgjGNDPDCo4tNqhFMymIAdJCcykFbdhYr4X6vD7IlMxY0t3/k6Pfup68YNkMTpRfLKTRuKDmnQ==", + "requires": { + "@types/events": "1.2.0", + "@types/node": "9.6.8" + } + }, + "ajv": { + "version": "5.5.2", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz", + "integrity": "sha1-c7Xuyj+rZT49P5Qis0GtQiBdyWU=", + "requires": { + "co": "4.6.0", + "fast-deep-equal": "1.1.0", + "fast-json-stable-stringify": "2.0.0", + "json-schema-traverse": "0.3.1" + } + }, + "asn1": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz", + "integrity": "sha1-2sh4dxPJlmhJ/IGAd36+nB3fO4Y=" + }, + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" + }, + "async-limiter": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz", + "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg==" + }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" + }, + "aws-sign2": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" + }, + "aws4": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.7.0.tgz", + "integrity": "sha512-32NDda82rhwD9/JBCCkB+MRYDp0oSvlo2IL6rQWA10PQi7tDUM3eqMSltXmY+Oyl/7N3P3qNtAlv7X0d9bI28w==" + }, + "bcrypt-pbkdf": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz", + "integrity": "sha1-Y7xdy2EzG5K8Bf1SiVPDNGKgb40=", + "optional": true, + "requires": { + "tweetnacl": "0.14.5" + } + }, + "boom": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/boom/-/boom-4.3.1.tgz", + "integrity": "sha1-T4owBctKfjiJ90kDD9JbluAdLjE=", + "requires": { + "hoek": "4.2.1" + } + }, + "cacheable-request": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-2.1.4.tgz", + "integrity": "sha1-DYCIAbY0KtM8kd+dC0TcCbkeXD0=", + "requires": { + "clone-response": "1.0.2", + "get-stream": "3.0.0", + "http-cache-semantics": "3.8.1", + "keyv": "3.0.0", + "lowercase-keys": "1.0.0", + "normalize-url": "2.0.1", + "responselike": "1.0.2" + }, + "dependencies": { + "lowercase-keys": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.0.tgz", + "integrity": "sha1-TjNms55/VFfjXxMkvfb4jQv8cwY=" + } + } + }, + "caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" + }, + "clone-response": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.2.tgz", + "integrity": "sha1-0dyXOSAxTfZ/vrlCI7TuNQI56Ws=", + "requires": { + "mimic-response": "1.0.0" + } + }, + "co": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", + "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=" + }, + "combined-stream": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.6.tgz", + "integrity": "sha1-cj599ugBrFYTETp+RFqbactjKBg=", + "requires": { + "delayed-stream": "1.0.0" + } + }, + "core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, + "cryptiles": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-3.1.2.tgz", + "integrity": "sha1-qJ+7Ig9c4l7FboxKqKT9e1sNKf4=", + "requires": { + "boom": "5.2.0" + }, + "dependencies": { + "boom": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/boom/-/boom-5.2.0.tgz", + "integrity": "sha512-Z5BTk6ZRe4tXXQlkqftmsAUANpXmuwlsF5Oov8ThoMbQRzdGTA1ngYRW160GexgOgjsFOKJz0LYhoNi+2AMBUw==", + "requires": { + "hoek": "4.2.1" + } + } + } + }, + "dashdash": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", + "requires": { + "assert-plus": "1.0.0" + } + }, + "decode-uri-component": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", + "integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=" + }, + "decompress-response": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", + "integrity": "sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=", + "requires": { + "mimic-response": "1.0.0" + } + }, + "define-properties": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.2.tgz", + "integrity": "sha1-g6c/L+pWmJj7c3GTyPhzyvbUXJQ=", + "requires": { + "foreach": "2.0.5", + "object-keys": "1.0.11" + } + }, + "delay": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/delay/-/delay-2.0.0.tgz", + "integrity": "sha1-kRLq3APk7H4AKXM3iW8nO72R+uU=", + "requires": { + "p-defer": "1.0.0" + } + }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" + }, + "dotenv": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-5.0.1.tgz", + "integrity": "sha512-4As8uPrjfwb7VXC+WnLCbXK7y+Ueb2B3zgNCePYfhxS1PYeaO1YTeplffTEcbfLhvFNGLAz90VvJs9yomG7bow==" + }, + "duplexer3": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz", + "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=" + }, + "ecc-jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz", + "integrity": "sha1-D8c6ntXw1Tw4GTOYUj735UN3dQU=", + "optional": true, + "requires": { + "jsbn": "0.1.1" + } + }, + "es-abstract": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.11.0.tgz", + "integrity": "sha512-ZnQrE/lXTTQ39ulXZ+J1DTFazV9qBy61x2bY071B+qGco8Z8q1QddsLdt/EF8Ai9hcWH72dWS0kFqXLxOxqslA==", + "requires": { + "es-to-primitive": "1.1.1", + "function-bind": "1.1.1", + "has": "1.0.1", + "is-callable": "1.1.3", + "is-regex": "1.0.4" + } + }, + "es-to-primitive": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.1.1.tgz", + "integrity": "sha1-RTVSSKiJeQNLZ5Lhm7gfK3l13Q0=", + "requires": { + "is-callable": "1.1.3", + "is-date-object": "1.0.1", + "is-symbol": "1.0.1" + } + }, + "eventemitter3": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.0.tgz", + "integrity": "sha512-ivIvhpq/Y0uSjcHDcOIccjmYjGLcP09MFGE7ysAwkAvkXfpZlC985pH2/ui64DKazbTW/4kN3yqozUxlXzI6cA==" + }, + "extend": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.1.tgz", + "integrity": "sha1-p1Xqe8Gt/MWjHOfnYtuq3F5jZEQ=" + }, + "extsprintf": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" + }, + "fast-deep-equal": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz", + "integrity": "sha1-wFNHeBfIa1HaqFPIHgWbcz0CNhQ=" + }, + "fast-json-stable-stringify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", + "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" + }, + "finity": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/finity/-/finity-0.5.4.tgz", + "integrity": "sha512-3l+5/1tuw616Lgb0QBimxfdd2TqaDGpfCBpfX6EqtFmqUV3FtQnVEX4Aa62DagYEqnsTIjZcTfbq9msDbXYgyA==" + }, + "foreach": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.5.tgz", + "integrity": "sha1-C+4AUBiusmDQo6865ljdATbsG5k=" + }, + "forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" + }, + "form-data": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.2.tgz", + "integrity": "sha1-SXBJi+YEwgwAXU9cI67NIda0kJk=", + "requires": { + "asynckit": "0.4.0", + "combined-stream": "1.0.6", + "mime-types": "2.1.18" + } + }, + "from2": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz", + "integrity": "sha1-i/tVAr3kpNNs/e6gB/zKIdfjgq8=", + "requires": { + "inherits": "2.0.3", + "readable-stream": "2.3.6" + } + }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + }, + "get-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=" + }, + "getpass": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", + "requires": { + "assert-plus": "1.0.0" + } + }, + "got": { + "version": "8.3.1", + "resolved": "https://registry.npmjs.org/got/-/got-8.3.1.tgz", + "integrity": "sha512-tiLX+bnYm5A56T5N/n9Xo89vMaO1mrS9qoDqj3u/anVooqGozvY/HbXzEpDfbNeKsHCBpK40gSbz8wGYSp3i1w==", + "requires": { + "@sindresorhus/is": "0.7.0", + "cacheable-request": "2.1.4", + "decompress-response": "3.3.0", + "duplexer3": "0.1.4", + "get-stream": "3.0.0", + "into-stream": "3.1.0", + "is-retry-allowed": "1.1.0", + "isurl": "1.0.0", + "lowercase-keys": "1.0.1", + "mimic-response": "1.0.0", + "p-cancelable": "0.4.1", + "p-timeout": "2.0.1", + "pify": "3.0.0", + "safe-buffer": "5.1.1", + "timed-out": "4.0.1", + "url-parse-lax": "3.0.0", + "url-to-options": "1.0.1" + }, + "dependencies": { + "p-cancelable": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-0.4.1.tgz", + "integrity": "sha512-HNa1A8LvB1kie7cERyy21VNeHb2CWJJYqyyC2o3klWFfMGlFmWv2Z7sFgZH8ZiaYL95ydToKTFVXgMV/Os0bBQ==" + } + } + }, + "har-schema": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" + }, + "har-validator": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.0.3.tgz", + "integrity": "sha1-ukAsJmGU8VlW7xXg/PJCmT9qff0=", + "requires": { + "ajv": "5.5.2", + "har-schema": "2.0.0" + } + }, + "has": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.1.tgz", + "integrity": "sha1-hGFzP1OLCDfJNh45qauelwTcLyg=", + "requires": { + "function-bind": "1.1.1" + } + }, + "has-symbol-support-x": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/has-symbol-support-x/-/has-symbol-support-x-1.4.2.tgz", + "integrity": "sha512-3ToOva++HaW+eCpgqZrCfN51IPB+7bJNVT6CUATzueB5Heb8o6Nam0V3HG5dlDvZU1Gn5QLcbahiKw/XVk5JJw==" + }, + "has-to-string-tag-x": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/has-to-string-tag-x/-/has-to-string-tag-x-1.4.1.tgz", + "integrity": "sha512-vdbKfmw+3LoOYVr+mtxHaX5a96+0f3DljYd8JOqvOLsf5mw2Otda2qCDT9qRqLAhrjyQ0h7ual5nOiASpsGNFw==", + "requires": { + "has-symbol-support-x": "1.4.2" + } + }, + "hawk": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/hawk/-/hawk-6.0.2.tgz", + "integrity": "sha512-miowhl2+U7Qle4vdLqDdPt9m09K6yZhkLDTWGoUiUzrQCn+mHHSmfJgAyGaLRZbPmTqfFFjRV1QWCW0VWUJBbQ==", + "requires": { + "boom": "4.3.1", + "cryptiles": "3.1.2", + "hoek": "4.2.1", + "sntp": "2.1.0" + } + }, + "hoek": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/hoek/-/hoek-4.2.1.tgz", + "integrity": "sha512-QLg82fGkfnJ/4iy1xZ81/9SIJiq1NGFUMGs6ParyjBZr6jW2Ufj/snDqTHixNlHdPNwN2RLVD0Pi3igeK9+JfA==" + }, + "http-cache-semantics": { + "version": "3.8.1", + "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz", + "integrity": "sha512-5ai2iksyV8ZXmnZhHH4rWPoxxistEexSi5936zIQ1bnNTW5VnA85B6P/VpXiRM017IgRvb2kKo1a//y+0wSp3w==" + }, + "http-signature": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", + "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", + "requires": { + "assert-plus": "1.0.0", + "jsprim": "1.4.1", + "sshpk": "1.14.1" + } + }, + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + }, + "into-stream": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/into-stream/-/into-stream-3.1.0.tgz", + "integrity": "sha1-lvsKk2wSur1v8XUqF9BWFqvQlMY=", + "requires": { + "from2": "2.3.0", + "p-is-promise": "1.1.0" + } + }, + "is-callable": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.3.tgz", + "integrity": "sha1-hut1OSgF3cM69xySoO7fdO52BLI=" + }, + "is-date-object": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz", + "integrity": "sha1-mqIOtq7rv/d/vTPnTKAbM1gdOhY=" + }, + "is-object": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-object/-/is-object-1.0.1.tgz", + "integrity": "sha1-iVJojF7C/9awPsyF52ngKQMINHA=" + }, + "is-plain-obj": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz", + "integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4=" + }, + "is-regex": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz", + "integrity": "sha1-VRdIm1RwkbCTDglWVM7SXul+lJE=", + "requires": { + "has": "1.0.1" + } + }, + "is-retry-allowed": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-1.1.0.tgz", + "integrity": "sha1-EaBgVotnM5REAz0BJaYaINVk+zQ=" + }, + "is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=" + }, + "is-symbol": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.1.tgz", + "integrity": "sha1-PMWfAAJRlLarLjjbrmaJJWtmBXI=" + }, + "is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" + }, + "isurl": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isurl/-/isurl-1.0.0.tgz", + "integrity": "sha512-1P/yWsxPlDtn7QeRD+ULKQPaIaN6yF368GZ2vDfv0AL0NwpStafjWCDDdn0k8wgFMWpVAqG7oJhxHnlud42i9w==", + "requires": { + "has-to-string-tag-x": "1.4.1", + "is-object": "1.0.1" + } + }, + "jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=", + "optional": true + }, + "json-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.0.tgz", + "integrity": "sha1-Wx85evx11ne96Lz8Dkfh+aPZqJg=" + }, + "json-schema": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" + }, + "json-schema-traverse": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz", + "integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A=" + }, + "json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" + }, + "jsprim": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", + "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "requires": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.2.3", + "verror": "1.10.0" + } + }, + "keyv": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-3.0.0.tgz", + "integrity": "sha512-eguHnq22OE3uVoSYG0LVWNP+4ppamWr9+zWBe1bsNcovIMy6huUJFPgy4mGwCd/rnl3vOLGW1MTlu4c57CT1xA==", + "requires": { + "json-buffer": "3.0.0" + } + }, + "loglevel": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.6.1.tgz", + "integrity": "sha1-4PyVEztu8nbNyIh82vJKpvFW+Po=" + }, + "lowercase-keys": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz", + "integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA==" + }, + "mime-db": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.33.0.tgz", + "integrity": "sha512-BHJ/EKruNIqJf/QahvxwQZXKygOQ256myeN/Ew+THcAa5q+PjyTTMMeNQC4DZw5AwfvelsUrA6B67NKMqXDbzQ==" + }, + "mime-types": { + "version": "2.1.18", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.18.tgz", + "integrity": "sha512-lc/aahn+t4/SWV/qcmumYjymLsWfN3ELhpmVuUFjgsORruuZPVSwAQryq+HHGvO/SI2KVX26bx+En+zhM8g8hQ==", + "requires": { + "mime-db": "1.33.0" + } + }, + "mimic-response": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.0.tgz", + "integrity": "sha1-3z02Uqc/3ta5sLJBRub9BSNTRY4=" + }, + "minimist": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" + }, + "normalize-url": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-2.0.1.tgz", + "integrity": "sha512-D6MUW4K/VzoJ4rJ01JFKxDrtY1v9wrgzCX5f2qj/lzH1m/lW6MhUZFKerVsnyjOhOsYzI9Kqqak+10l4LvLpMw==", + "requires": { + "prepend-http": "2.0.0", + "query-string": "5.1.1", + "sort-keys": "2.0.0" + } + }, + "oauth-sign": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz", + "integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM=" + }, + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" + }, + "object-keys": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.0.11.tgz", + "integrity": "sha1-xUYBd4rVYPEULODgG8yotW0TQm0=" + }, + "object.entries": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/object.entries/-/object.entries-1.0.4.tgz", + "integrity": "sha1-G/mk3SKI9bM/Opk9JXZh8F0WGl8=", + "requires": { + "define-properties": "1.1.2", + "es-abstract": "1.11.0", + "function-bind": "1.1.1", + "has": "1.0.1" + } + }, + "object.getownpropertydescriptors": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.0.3.tgz", + "integrity": "sha1-h1jIRvW0B62rDyNuCYbxSwUcqhY=", + "requires": { + "define-properties": "1.1.2", + "es-abstract": "1.11.0" + } + }, + "object.values": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.0.4.tgz", + "integrity": "sha1-5STaCbT2b/Bd9FdUbscqyZ8TBpo=", + "requires": { + "define-properties": "1.1.2", + "es-abstract": "1.11.0", + "function-bind": "1.1.1", + "has": "1.0.1" + } + }, + "p-cancelable": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-0.3.0.tgz", + "integrity": "sha512-RVbZPLso8+jFeq1MfNvgXtCRED2raz/dKpacfTNxsx6pLEpEomM7gah6VeHSYV3+vo0OAi4MkArtQcWWXuQoyw==" + }, + "p-defer": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-1.0.0.tgz", + "integrity": "sha1-n26xgvbJqozXQwBKfU+WsZaw+ww=" + }, + "p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=" + }, + "p-is-promise": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/p-is-promise/-/p-is-promise-1.1.0.tgz", + "integrity": "sha1-nJRWmJ6fZYgBewQ01WCXZ1w9oF4=" + }, + "p-queue": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-2.4.2.tgz", + "integrity": "sha512-n8/y+yDJwBjoLQe1GSJbbaYQLTI7QHNZI2+rpmCDbe++WLf9HC3gf6iqj5yfPAV71W4UF3ql5W1+UBPXoXTxng==" + }, + "p-retry": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-1.0.0.tgz", + "integrity": "sha1-OSczKkt9cCabU1UVEX/FR9oaaWg=", + "requires": { + "retry": "0.10.1" + } + }, + "p-timeout": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-2.0.1.tgz", + "integrity": "sha512-88em58dDVB/KzPEx1X0N3LwFfYZPyDc4B6eF38M1rk9VTZMbxXXgjugz8mmwpS9Ox4BDZ+t6t3QP5+/gazweIA==", + "requires": { + "p-finally": "1.0.0" + } + }, + "performance-now": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" + }, + "pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=" + }, + "prepend-http": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-2.0.0.tgz", + "integrity": "sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc=" + }, + "process-nextick-args": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", + "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" + }, + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + }, + "qs": { + "version": "6.5.1", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz", + "integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A==" + }, + "query-string": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/query-string/-/query-string-5.1.1.tgz", + "integrity": "sha512-gjWOsm2SoGlgLEdAGt7a6slVOk9mGiXmPFMqrEhLQ68rhQuBnpfs3+EmlvqKyxnCo9/PPlF+9MtY02S1aFg+Jw==", + "requires": { + "decode-uri-component": "0.2.0", + "object-assign": "4.1.1", + "strict-uri-encode": "1.1.0" + } + }, + "readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "1.0.0", + "process-nextick-args": "2.0.0", + "safe-buffer": "5.1.1", + "string_decoder": "1.1.1", + "util-deprecate": "1.0.2" + } + }, + "request": { + "version": "2.85.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.85.0.tgz", + "integrity": "sha512-8H7Ehijd4js+s6wuVPLjwORxD4zeuyjYugprdOXlPSqaApmL/QOy+EB/beICHVCHkGMKNh5rvihb5ov+IDw4mg==", + "requires": { + "aws-sign2": "0.7.0", + "aws4": "1.7.0", + "caseless": "0.12.0", + "combined-stream": "1.0.6", + "extend": "3.0.1", + "forever-agent": "0.6.1", + "form-data": "2.3.2", + "har-validator": "5.0.3", + "hawk": "6.0.2", + "http-signature": "1.2.0", + "is-typedarray": "1.0.0", + "isstream": "0.1.2", + "json-stringify-safe": "5.0.1", + "mime-types": "2.1.18", + "oauth-sign": "0.8.2", + "performance-now": "2.1.0", + "qs": "6.5.1", + "safe-buffer": "5.1.1", + "stringstream": "0.0.5", + "tough-cookie": "2.3.4", + "tunnel-agent": "0.6.0", + "uuid": "3.2.1" + } + }, + "responselike": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/responselike/-/responselike-1.0.2.tgz", + "integrity": "sha1-kYcg7ztjHFZCvgaPFa3lpG9Loec=", + "requires": { + "lowercase-keys": "1.0.1" + } + }, + "retry": { + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.10.1.tgz", + "integrity": "sha1-52OI0heZLCUnUCQdPTlW/tmNj/Q=" + }, + "safe-buffer": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", + "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==" + }, + "sntp": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/sntp/-/sntp-2.1.0.tgz", + "integrity": "sha512-FL1b58BDrqS3A11lJ0zEdnJ3UOKqVxawAkF3k7F0CVN7VQ34aZrV+G8BZ1WC9ZL7NyrwsW0oviwsWDgRuVYtJg==", + "requires": { + "hoek": "4.2.1" + } + }, + "sort-keys": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/sort-keys/-/sort-keys-2.0.0.tgz", + "integrity": "sha1-ZYU1WEhh7JfXMNbPQYIuH1ZoQSg=", + "requires": { + "is-plain-obj": "1.1.0" + } + }, + "sshpk": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.14.1.tgz", + "integrity": "sha1-Ew9Zde3a2WPx1W+SuaxsUfqfg+s=", + "requires": { + "asn1": "0.2.3", + "assert-plus": "1.0.0", + "bcrypt-pbkdf": "1.0.1", + "dashdash": "1.14.1", + "ecc-jsbn": "0.1.1", + "getpass": "0.1.7", + "jsbn": "0.1.1", + "tweetnacl": "0.14.5" + } + }, + "strict-uri-encode": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz", + "integrity": "sha1-J5siXfHVgrH1TmWt3UNS4Y+qBxM=" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "5.1.1" + } + }, + "stringstream": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz", + "integrity": "sha1-TkhM1N5aC7vuGORjB3EKioFiGHg=" + }, + "timed-out": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-4.0.1.tgz", + "integrity": "sha1-8y6srFoXW+ol1/q1Zas+2HQe9W8=" + }, + "tough-cookie": { + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.4.tgz", + "integrity": "sha512-TZ6TTfI5NtZnuyy/Kecv+CnoROnyXn2DN97LontgQpCwsX2XyLYCC0ENhYkehSOwAp8rTQKc/NUIF7BkQ5rKLA==", + "requires": { + "punycode": "1.4.1" + } + }, + "tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "requires": { + "safe-buffer": "5.1.1" + } + }, + "tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", + "optional": true + }, + "url-join": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/url-join/-/url-join-4.0.0.tgz", + "integrity": "sha1-TTNA6AfTdzvamZH4MFrNzCpmXSo=" + }, + "url-parse-lax": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-3.0.0.tgz", + "integrity": "sha1-FrXK/Afb42dsGxmZF3gj1lA6yww=", + "requires": { + "prepend-http": "2.0.0" + } + }, + "url-to-options": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/url-to-options/-/url-to-options-1.0.1.tgz", + "integrity": "sha1-FQWgOiiaSMvXpDTvuu7FBV9WM6k=" + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + }, + "uuid": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.2.1.tgz", + "integrity": "sha512-jZnMwlb9Iku/O3smGWvZhauCf6cvvpKi4BKRiliS3cxnI+Gz9j5MEpTz2UFuXiKPJocb7gnsLHwiS05ige5BEA==" + }, + "verror": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "requires": { + "assert-plus": "1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "1.3.0" + } + }, + "ws": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-4.1.0.tgz", + "integrity": "sha512-ZGh/8kF9rrRNffkLFV4AzhvooEclrOH0xaugmqGsIfFgOE/pIz4fMc4Ef+5HSQqTEug2S9JZIWDR47duDSLfaA==", + "requires": { + "async-limiter": "1.0.0", + "safe-buffer": "5.1.1" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..886b950 --- /dev/null +++ b/package.json @@ -0,0 +1,22 @@ +{ + "name": "samplernn", + "version": "1.0.0", + "description": "samplernn", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git@asdf.us:samplernn.git" + }, + "author": "jules <jules@asdf.us>", + "license": "UNLICENSED", + "private": true, + "dependencies": { + "@slack/client": "^4.2.0", + "dotenv": "^5.0.1", + "minimist": "^1.2.0", + "request": "^2.85.0" + } +} diff --git a/process_fns b/process_fns new file mode 100755 index 0000000..cef738d --- /dev/null +++ b/process_fns @@ -0,0 +1,140 @@ +# workon samplernn + +function generate () { + exp_name=${1%.*} + n_samples=$2 + sample_rate=$3 + duration=$4 + let sample_length=$3*$4 + + sleep 0.1 + echo "" + echo "###################################################" + echo "###################################################" + echo "###################################################" + echo "" + echo ">> generating $exp_name" + echo "" + echo "###################################################" + echo "###################################################" + echo "###################################################" + echo "" + python generate.py \ + --exp $exp_name --dataset $exp_name \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --n_samples $n_samples \ + --epoch_limit 1 \ + --resume True +} +function gen_len () { + exp_name=${1%.*} + n_samples=$2 + sample_rate=$3 + sample_length=$4 + + sleep 0.1 + echo "" + echo "###################################################" + echo "###################################################" + echo "###################################################" + echo "" + echo ">> generating $exp_name" + echo "" + echo "###################################################" + echo "###################################################" + echo "###################################################" + echo "" + python generate.py \ + --exp $exp_name --dataset $exp_name \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --n_samples $n_samples \ + --epoch_limit 1 \ + --resume True +} +function run () { + exp_name=${1%.*} + epoch_limit=$2 + n_samples=$3 + sample_rate=$4 + duration=$5 + let sample_length=$4*$5 + + sleep 0.1 + echo "" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo "" + echo ">> running $exp_name" + echo "" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo "" + python train.py \ + --exp $exp_name --dataset $exp_name \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --n_samples $n_samples \ + --keep_old_checkpoints False \ + --epoch_limit $epoch_limit \ + --resume True +} +function standard () { + dataset=$1 + run $1 6 6 44100 5 + ./latest.pl -l $1 +} +function quick () { + dataset=$1 + run $1 4 6 44100 5 + ./latest.pl -l $1 +} +function half () { + dataset=$1 + run $1 2 6 44100 5 + ./latest.pl -l $1 +} +function fast () { + dataset=$1 + run $1 1 6 44100 10 + ./latest.pl -l $1 +} +function emph () { + dataset=$1 + run $1 1 6 44100 10 + run $1 3 6 44100 5 + ./latest.pl -l $1 +} + +function start () { + dataset=$1 + cd datasets/ + perl dataset.pl "${dataset}.wav" + cd .. + quick $dataset +} +function start_standard () { + dataset=$1 + cd datasets/ + perl dataset.pl "${dataset}.wav" + cd .. + standard $dataset +} + diff --git a/results/crossfade_cat.sh b/results/crossfade_cat.sh new file mode 100755 index 0000000..13a8e90 --- /dev/null +++ b/results/crossfade_cat.sh @@ -0,0 +1,129 @@ +#!/bin/bash +# +# crossfade_cat.sh +# +# Concatenates two files together with a crossfade of $1 seconds. +# Filenames are specified as $2 and $3. +# +# $4 is optional and specifies if a fadeout should be performed on +# first file. +# $5 is optional and specifies if a fadein should be performed on +# second file. +# +# Example: crossfade_cat.sh 10 infile1.wav infile2.wav auto auto +# +# By default, the script attempts to guess if the audio files +# already have a fadein/out on them or if they just have really +# low volumes that won't cause clipping when mixxing. If this +# is not detected then the script will perform a fade in/out to +# prevent clipping. +# +# The user may specify "yes" or "no" to force the fade in/out +# to occur. They can also specify "auto" which is the default. +# +# Crossfaded file is created as "mix.wav". +# +# Original script from Kester Clegg. Mods by Chris Bagwell to show +# more examples of sox features. +# + +SOX=sox +SOXI=soxi + +if [ "$3" == "" ]; then + echo "Usage: $0 crossfade_seconds first_file second_file [ fadeout ] [ fadein ]" + echo + echo "If a fadeout or fadein is not desired then specify \"no\" for that option. \"yes\" will force a fade and \"auto\" will try to detect if a fade should occur." + echo + echo "Example: $0 10 infile1.wav infile2.wav auto auto" + exit 1 +fi + +fade_length=$1 +first_file=$2 +second_file=$3 + +fade_first="auto" +if [ "$4" != "" ]; then + fade_first=$4 +fi + +fade_second="auto" +if [ "$5" != "" ]; then + fade_second=$5 +fi + +fade_first_opts= +if [ "$fade_first" != "no" ]; then + fade_first_opts="fade t 0 0:0:$fade_length 0:0:$fade_length" +fi + +fade_second_opts= +if [ "$fade_second" != "no" ]; then + fade_second_opts="fade t 0:0:$fade_length" +fi + +echo "crossfade and concatenate files" +echo +echo "Finding length of $first_file..." +first_length=`$SOX "$first_file" 2>&1 -n stat | grep Length | cut -d : -f 2 | cut -f 1` +echo "Length is $first_length seconds" + +trim_length=`echo "$first_length - $fade_length" | bc` + +# Get crossfade section from first file and optionally do the fade out +echo "Obtaining $fade_length seconds of fade out portion from $first_file..." +$SOX "$first_file" -e signed-integer -b 16 fadeout1.wav trim $trim_length + +# When user specifies "auto" try to guess if a fadeout is needed. +# "RMS amplitude" from the stat effect is effectively an average +# value of samples for the whole fade length file. If it seems +# quite then assume a fadeout has already been done. An RMS value +# of 0.1 was just obtained from trail and error. +if [ "$fade_first" == "auto" ]; then + RMS=`$SOX fadeout1.wav 2>&1 -n stat | grep RMS | grep amplitude | cut -d : -f 2 | cut -f 1` + should_fade=`echo "$RMS > 0.1" | bc` + if [ $should_fade == 0 ]; then + echo "Auto mode decided not to fadeout with RMS of $RMS" + fade_first_opts="" + else + echo "Auto mode will fadeout" + fi +fi + +$SOX fadeout1.wav fadeout2.wav $fade_first_opts + +# Get the crossfade section from the second file and optionally do the fade in +echo "Obtaining $fade_length seconds of fade in portion from $second_file..." +$SOX "$second_file" -e signed-integer -b 16 fadein1.wav trim 0 $fade_length + +# For auto, do similar thing as for fadeout. +if [ "$fade_second" == "auto" ]; then + RMS=`$SOX fadein1.wav 2>&1 -n stat | grep RMS | grep amplitude | cut -d : -f 2 | cut -f 1` + should_fade=`echo "$RMS > 0.1" | bc` + if [ $should_fade == 0 ]; then + echo "Auto mode decided not to fadein with RMS of $RMS" + fade_second_opts="" + else + echo "Auto mode will fadein" + fi +fi + +$SOX fadein1.wav fadein2.wav $fade_second_opts + +# Mix the crossfade files together at full volume +echo "Crossfading..." +$SOX -m -v 1.0 fadeout2.wav -v 1.0 fadein2.wav crossfade.wav + +echo "Trimming off crossfade sections from original files..." + +$SOX "$first_file" -e signed-integer -b 16 song1.wav trim 0 $trim_length +$SOX "$second_file" -e signed-integer -b 16 song2.wav trim $fade_length +$SOX song1.wav crossfade.wav song2.wav mix.wav + +echo -e "Removing temporary files...\n" +rm fadeout1.wav fadeout2.wav fadein1.wav fadein2.wav crossfade.wav song1.wav song2.wav +mins=`echo "$trim_length / 60" | bc` +secs=`echo "$trim_length % 60" | bc` +echo "The crossfade in mix.wav occurs at around $mins mins $secs secs" + diff --git a/results/loop.sh b/results/loop.sh new file mode 100755 index 0000000..beb1286 --- /dev/null +++ b/results/loop.sh @@ -0,0 +1,20 @@ +rm fades.txt +rm files.txt +rm mix.sh + +ITER=0 +NEXT=1 +for i in `ls -v *.wav` +do +echo "[0$ITER][$NEXT]acrossfade=ns=5:o=1:c1=tri:c2=tri[0$NEXT];" >> fades.txt +printf '\-i %s ' $i >> files.txt +ITER=$(expr $ITER + 1) +NEXT=$(expr $NEXT + 1) +done + +printf "ffmpeg " >> mix.sh +cat files.txt >> mix.sh +printf " -filter_complex \"" >> mix.sh +cat fades.txt >> mix.sh +echo '\" out.wav' >> mix.sh + diff --git a/results/xfade.sh b/results/xfade.sh new file mode 100755 index 0000000..55d343d --- /dev/null +++ b/results/xfade.sh @@ -0,0 +1,25 @@ +crossfade_dur=1 +i=0 +limit=10000 + +for file in `ls *.wav | sort -V` +do + i=$((i+1)) + if [ $i -eq $limit ] + then + break + fi + + if [ $i -eq 1 ] + then + sox $file mix.wav highpass 10 + else + # ../../crossfade_cat.sh $crossfade_dur mix.wav $file yes yes + # echo $file + sox $file tmp.wav highpass 10 + sox mix.wav tmp.wav out.wav splice $(soxi -D mix.wav),0.01 + mv out.wav mix.wav + rm tmp.wav + fi +done + @@ -0,0 +1,88 @@ +function generate () { + exp_name=$1 + n_samples=$2 + sample_rate=$3 + duration=$4 + let sample_length=$3*$4 + + echo "" + echo "###################################################" + echo "###################################################" + echo "###################################################" + echo "" + echo ">> generating $exp_name" + echo "" + echo "###################################################" + echo "###################################################" + echo "###################################################" + echo "" + python generate.py \ + --exp $exp_name --dataset $exp_name \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --n_samples $n_samples \ + --epoch_limit 1 \ + --resume True +} +function run () { + exp_name=$1 + epoch_limit=$2 + n_samples=$3 + sample_rate=$4 + duration=$5 + let sample_length=$4*$5 + + echo "" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo "" + echo ">> running $exp_name" + echo "" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo "" + python train.py \ + --exp $exp_name --dataset $exp_name \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --n_samples $n_samples \ + --keep_old_checkpoints False \ + --epoch_limit $epoch_limit \ + --resume True +} +function standard () { + dataset=$1 + run $1 6 6 44100 5 +} +function quick () { + dataset=$1 + run $1 4 6 44100 5 +} +function fast () { + dataset=$1 + run $1 1 6 44100 10 +} + +quick 01_snapping_clean +quick 01_snapping_compress +quick 01_snapping_highpass + +quick 01_snapping_clean +quick 01_snapping_compress +quick 01_snapping_highpass + +quick 01_snapping_clean +quick 01_snapping_compress +quick 01_snapping_highpass + diff --git a/spawn.js b/spawn.js new file mode 100644 index 0000000..13ee81b --- /dev/null +++ b/spawn.js @@ -0,0 +1,173 @@ +require('dotenv').load(); + +const { RTMClient, WebClient } = require('@slack/client'); + +// An access token (from your Slack app or custom integration - usually xoxb) +const token = process.env.SLACK_CLI_TOKEN; + +class SpawnClient { + constructor(){ + console.log('signing in', token) + this.web = new WebClient(token); + this.rtm = new RTMClient(token) + this.rtm.on('connected', () => console.log('connected')) + this.rtm.on('authenticated', () => console.log('authenticated')) + this.rtm.start(); + + this.listen() + this.list() + } + listen(){ + console.log('listening') + this.rtm.on('message', (message) => { + console.log(message) + if ( (message.subtype && message.subtype === 'bot_message') || + (!message.subtype && message.user === this.rtm.activeUserId) ) { + return; + } + switch (message.subtype) { + case 'file_share': + this.processFile(message.file) + break + case 'file_comment': + this.processFileCommand(message.file, message.comment) + break + // case 'file_mention': + // break + default: + console.log(`(channel:${message.channel}) ${message.user} says: ${message.text}`); + break + } + }) + } + list(){ + console.log('listing groups..') + this.web.groups.list().then((res) => { + console.log("____________\n") + console.log(res) + + const group = res.groups[0] + + if (group) { + this.group = group + console.log('we are in', group) + } else { + console.log('This bot does not belong to any private group, invite it to at least one and try again'); + } + }); + } + + postMessageToGroup(msg){ + if (! this.group) { + console.log('not in a group!') + return + } + rtm.sendMessage(msg, this.group.id) + .then((msg) => console.log(`ts:${msg.ts} <${group.name}> ${msg}`)) + .catch(console.error); + } + + processFile(){ + console.log(message.text) + console.log(`download: ${message.file.url_private_download}`) + } + + processFileCommand(file, comment){ + const msg = comment.comment + console.log(`comment on file: ${msg}`) + } +} + +const client = new SpawnClient () + +// This argument can be a channel ID, a DM ID, a MPDM ID, or a group ID +// const conversationId = 'C1232456'; + +// The RTM client can send simple string messages +// rtm.sendMessage('Hello there', conversationId) +// .then((res) => { +// // `res` contains information about the posted message +// console.log('Message sent: ', res.ts); +// }) +// .catch(console.error); + + + +// { +// "type": "message", +// "subtype": "file_share", +// "ts": "1358877455.000010", +// "text": "<@cal> uploaded a file: <https:...7.png|7.png>", +// "file": {...}, +// "user": "U2147483697", +// "upload": true +// } + +// { +// "type": "message", +// "subtype": "file_comment", +// "ts": "1361482916.000003", +// "text": "<@cal> commented on a file: ...", +// "file": {}, +// "comment": {} +// } + +// { +// "id" : "F2147483862", +// "created" : 1356032811, +// "timestamp" : 1356032811, +// "name" : "file.htm", +// "title" : "My HTML file", +// "mimetype" : "text\/plain", +// "filetype" : "text", +// "pretty_type": "Text", +// "user" : "U2147483697", +// "mode" : "hosted", +// "editable" : true, +// "is_external": false, +// "external_type": "", +// "username": "", +// "size" : 12345, +// "url_private": "https:\/\/slack.com\/files-pri\/T024BE7LD-F024BERPE\/1.png", +// "url_private_download": "https:\/\/slack.com\/files-pri\/T024BE7LD-F024BERPE\/download\/1.png", +// "thumb_64": "https:\/\/slack-files.com\/files-tmb\/T024BE7LD-F024BERPE-c66246\/1_64.png", +// "thumb_80": "https:\/\/slack-files.com\/files-tmb\/T024BE7LD-F024BERPE-c66246\/1_80.png", +// "thumb_360": "https:\/\/slack-files.com\/files-tmb\/T024BE7LD-F024BERPE-c66246\/1_360.png", +// "thumb_360_gif": "https:\/\/slack-files.com\/files-tmb\/T024BE7LD-F024BERPE-c66246\/1_360.gif", +// "thumb_360_w": 100, +// "thumb_360_h": 100, +// "thumb_480": "https:\/\/slack-files.com\/files-tmb\/T024BE7LD-F024BERPE-c66246\/1_480.png", +// "thumb_480_w": 480, +// "thumb_480_h": 480, +// "thumb_160": "https:\/\/slack-files.com\/files-tmb\/T024BE7LD-F024BERPE-c66246\/1_160.png", +// "permalink": "https:\/\/tinyspeck.slack.com\/files\/cal\/F024BERPE\/1.png", +// "permalink_public" : "https:\/\/tinyspeck.slack.com\/T024BE7LD-F024BERPE-3f9216b62c", +// "edit_link": "https:\/\/tinyspeck.slack.com\/files\/cal\/F024BERPE\/1.png/edit", +// "preview": "<!DOCTYPE html>\n<html>\n<meta charset='utf-8'>", +// "preview_highlight": "<div class=\"sssh-code\"><div class=\"sssh-line\"><pre><!DOCTYPE html...", +// "lines": 123, +// "lines_more": 118, +// "is_public": true, +// "public_url_shared": false, +// "display_as_bot" : false, +// "channels": ["C024BE7LT", ...], +// "groups": ["G12345", ...], +// "ims": ["D12345", ...], +// "initial_comment": {...}, +// "num_stars": 7, +// "is_starred": true, +// "pinned_to": ["C024BE7LT", ...], +// "reactions": [ +// { +// "name": "astonished", +// "count": 3, +// "users": [ "U1", "U2", "U3" ] +// }, +// { +// "name": "facepalm", +// "count": 1034, +// "users": [ "U1", "U2", "U3", "U4", "U5" ] +// } +// ], +// "comments_count": 1 +// }
\ No newline at end of file @@ -256,7 +256,7 @@ def main(exp, frame_sizes, dataset, **params): init_comet(params, trainer) - trainer.run(params['epoch_limit']) + trainer.run(int(params['epoch_limit'])) if __name__ == '__main__': diff --git a/train_basic_22k.sh b/train_basic_22k.sh new file mode 100755 index 0000000..5104034 --- /dev/null +++ b/train_basic_22k.sh @@ -0,0 +1,15 @@ +exp_name=$1 +dataset=$2 +epoch_limit=$3 +python train.py \ + --exp $exp_name --dataset $exp_name \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 64 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate 22050 \ + --sample_length 110250 \ + --keep_old_checkpoints False \ + --epoch_limit $epoch_limit \ + --resume True + diff --git a/train_basic_32k.sh b/train_basic_32k.sh new file mode 100755 index 0000000..c4d1319 --- /dev/null +++ b/train_basic_32k.sh @@ -0,0 +1,15 @@ +exp_name=$1 +dataset=$2 +epoch_limit=$3 +python train.py \ + --exp $exp_name --dataset $exp_name \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 64 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate 32000 \ + --sample_length 160000 \ + --keep_old_checkpoints False \ + --epoch_limit $epoch_limit \ + --resume True + diff --git a/train_basic_44k.sh b/train_basic_44k.sh new file mode 100755 index 0000000..1908f5d --- /dev/null +++ b/train_basic_44k.sh @@ -0,0 +1,15 @@ +exp_name=$1 +dataset=$2 +epoch_limit=$3 +python train.py \ + --exp $exp_name --dataset $exp_name \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 64 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate 44100 \ + --sample_length 220500 \ + --keep_old_checkpoints False \ + --epoch_limit $epoch_limit \ + --resume True + diff --git a/train_drums.sh b/train_drums.sh new file mode 100755 index 0000000..2157fc7 --- /dev/null +++ b/train_drums.sh @@ -0,0 +1,44 @@ +function generate () { + exp_name=$1 + epoch_limit=$2 + sample_rate=$3 + sample_length=$4 + + echo ">> generating $exp_name" + python generate.py \ + --exp $exp_name --dataset $exp_name \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit $epoch_limit \ + --resume True +} +function run () { + exp_name=$1 + epoch_limit=$2 + sample_rate=$3 + sample_length=$4 + + echo ">> running $exp_name" + python train.py \ + --exp $exp_name --dataset $exp_name \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit $epoch_limit \ + --resume True +} + +#generate 44k_jlin3 64 44100 4134 +generate 44k_jlin4 64 44100 4134 +generate 44k_clouds2 64 44100 4134 +generate 44k_glassmix 64 44100 4134 + diff --git a/trainer/__init__.py b/trainer/__init__.py index 7e2ea18..1f39506 100644 --- a/trainer/__init__.py +++ b/trainer/__init__.py @@ -56,6 +56,15 @@ class Trainer(object): self.train() self.call_plugins('epoch', self.epochs) + def generate(self, epochs=1): + for q in self.plugin_queues.values(): + heapq.heapify(q) + + for self.epochs in range(self.epochs + 1, self.epochs + epochs + 1): + # self.train() + self.call_plugins('update', self.iterations, self.model) + self.call_plugins('epoch', self.epochs) + def train(self): for (self.iterations, data) in \ enumerate(self.dataset, self.iterations + 1): diff --git a/trainer/plugins.py b/trainer/plugins.py index f8c299b..0126870 100644 --- a/trainer/plugins.py +++ b/trainer/plugins.py @@ -141,7 +141,7 @@ class SaverPlugin(Plugin): class GeneratorPlugin(Plugin): - pattern = 'ep{}-s{}.wav' + pattern = 'd-{}-ep{}-s{}.wav' def __init__(self, samples_path, n_samples, sample_length, sample_rate): super().__init__([(1, 'epoch')]) @@ -159,7 +159,7 @@ class GeneratorPlugin(Plugin): for i in range(self.n_samples): write_wav( os.path.join( - self.samples_path, self.pattern.format(epoch_index, i + 1) + self.samples_path, self.pattern.format(int(time.time()), epoch_index, i + 1) ), samples[i, :], sr=self.sample_rate, norm=True ) @@ -168,7 +168,7 @@ class GeneratorPlugin(Plugin): class StatsPlugin(Plugin): data_file_name = 'stats.pkl' - plot_pattern = '{}.svg' + plot_pattern = 'd-{}-{}.svg' def __init__(self, results_path, iteration_fields, epoch_fields, plots): super().__init__([(1, 'iteration'), (1, 'epoch')]) @@ -252,7 +252,7 @@ class StatsPlugin(Plugin): pyplot.legend() pyplot.savefig( - os.path.join(self.results_path, self.plot_pattern.format(name)) + os.path.join(self.results_path, self.plot_pattern.format(int(time.time()), name)) ) @staticmethod diff --git a/upload.js b/upload.js new file mode 100755 index 0000000..1abc5e9 --- /dev/null +++ b/upload.js @@ -0,0 +1,54 @@ +#!/home/spawn/.nvm/versions/node/v9.11.1/bin/node + +const { execFile } = require('child_process'); + +require('dotenv').load(); +var request = require('request') +var path = require('path') +var fs = require('fs') +var argv = require('minimist')(process.argv.slice(2)); + +var request = request.defaults({jar: true}) + +var j = request.jar() +var cookie = request.cookie('bucky.sid=' + process.env.COOKIE) +j.setCookie(cookie, 'https://bucky.asdf.us/'); +var request = request.defaults({jar:j}) + +var id = process.env.THREAD_ID + +console.log(argv) + +if (!argv['_'] || ! argv['_'].length) { + console.error('not enough args!') + process.exit() +} + +var r = request.post('https://bucky.asdf.us/api/thread/' + id + '/comment', function (err, res, body) { + if (err) { + return console.error('Upload failed:', err); + } + // console.log(err, res, body) + console.log('\n') + console.log('Upload successful!') + console.log('\n') + console.log(body) + argv['_'].forEach(fn => { + const partz = fn.split('/') + const url = 'https://s3.amazonaws.com/i.asdf.us/bucky/data/' + id + '/' + partz[partz.length-1] + console.log(url) + const child = execFile('slack', ['chat', 'send', url, '#ai'], (error, stdout, stderr) => { + if (error) { + throw error; + } + console.log(stdout); + }); + }) +}) + +var form = r.form() +form.append('comment', '') +form.append('csrf', 'csrf') +argv['_'].forEach(fn => { + form.append('files', fs.createReadStream(path.join(__dirname, fn))) +}) |