From ea6e6ee1040fa85f743ab50b699fbeb04d9a0522 Mon Sep 17 00:00:00 2001 From: jules Date: Tue, 20 Mar 2018 23:35:18 +0100 Subject: scripts --- .gitignore | 8 + best-epoch.sh | 1 + bin/crossfade_cat.sh | 129 ++++++++++++++++ bin/loop.sh | 20 +++ bin/xfade.sh | 23 +++ datasets/count_subdirs.sh | 1 + datasets/generate.sh | 34 +++++ datasets/split44k.sh | 24 +++ datasets/spread.sh | 25 ++++ generate.py | 365 ++++++++++++++++++++++++++++++++++++++++++++++ glass_test2.sh | 85 +++++++++++ glass_test3.sh | 112 ++++++++++++++ glass_tests.sh | 202 +++++++++++++++++++++++++ kick_test.sh | 51 +++++++ mix.sh | 20 +++ train.py | 2 +- train_basic_22k.sh | 15 ++ train_basic_32k.sh | 15 ++ train_basic_44k.sh | 15 ++ train_drums.sh | 44 ++++++ train_test_generate.sh | 93 ++++++++++++ trainer/__init__.py | 9 ++ trainer/plugins.py | 8 +- 23 files changed, 1296 insertions(+), 5 deletions(-) create mode 100644 best-epoch.sh create mode 100755 bin/crossfade_cat.sh create mode 100755 bin/loop.sh create mode 100755 bin/xfade.sh create mode 100755 datasets/count_subdirs.sh create mode 100755 datasets/generate.sh create mode 100755 datasets/split44k.sh create mode 100755 datasets/spread.sh create mode 100644 generate.py create mode 100644 glass_test2.sh create mode 100644 glass_test3.sh create mode 100644 glass_tests.sh create mode 100644 kick_test.sh create mode 100755 mix.sh create mode 100644 train_basic_22k.sh create mode 100644 train_basic_32k.sh create mode 100644 train_basic_44k.sh create mode 100644 train_drums.sh create mode 100644 train_test_generate.sh diff --git a/.gitignore b/.gitignore index 9c17115..3bb05ff 100644 --- a/.gitignore +++ b/.gitignore @@ -92,3 +92,11 @@ ENV/ *~ *.swp *.swo + +*.wav +*.mp3 +*.aif +*.aiff + +results/ + diff --git a/best-epoch.sh b/best-epoch.sh new file mode 100644 index 0000000..230daa0 --- /dev/null +++ b/best-epoch.sh @@ -0,0 +1 @@ +ls -lat results/*/checkpoints/best* | head -n 20 diff --git a/bin/crossfade_cat.sh b/bin/crossfade_cat.sh new file mode 100755 index 0000000..13a8e90 --- /dev/null +++ b/bin/crossfade_cat.sh @@ -0,0 +1,129 @@ +#!/bin/bash +# +# crossfade_cat.sh +# +# Concatenates two files together with a crossfade of $1 seconds. +# Filenames are specified as $2 and $3. +# +# $4 is optional and specifies if a fadeout should be performed on +# first file. +# $5 is optional and specifies if a fadein should be performed on +# second file. +# +# Example: crossfade_cat.sh 10 infile1.wav infile2.wav auto auto +# +# By default, the script attempts to guess if the audio files +# already have a fadein/out on them or if they just have really +# low volumes that won't cause clipping when mixxing. If this +# is not detected then the script will perform a fade in/out to +# prevent clipping. +# +# The user may specify "yes" or "no" to force the fade in/out +# to occur. They can also specify "auto" which is the default. +# +# Crossfaded file is created as "mix.wav". +# +# Original script from Kester Clegg. Mods by Chris Bagwell to show +# more examples of sox features. +# + +SOX=sox +SOXI=soxi + +if [ "$3" == "" ]; then + echo "Usage: $0 crossfade_seconds first_file second_file [ fadeout ] [ fadein ]" + echo + echo "If a fadeout or fadein is not desired then specify \"no\" for that option. \"yes\" will force a fade and \"auto\" will try to detect if a fade should occur." + echo + echo "Example: $0 10 infile1.wav infile2.wav auto auto" + exit 1 +fi + +fade_length=$1 +first_file=$2 +second_file=$3 + +fade_first="auto" +if [ "$4" != "" ]; then + fade_first=$4 +fi + +fade_second="auto" +if [ "$5" != "" ]; then + fade_second=$5 +fi + +fade_first_opts= +if [ "$fade_first" != "no" ]; then + fade_first_opts="fade t 0 0:0:$fade_length 0:0:$fade_length" +fi + +fade_second_opts= +if [ "$fade_second" != "no" ]; then + fade_second_opts="fade t 0:0:$fade_length" +fi + +echo "crossfade and concatenate files" +echo +echo "Finding length of $first_file..." +first_length=`$SOX "$first_file" 2>&1 -n stat | grep Length | cut -d : -f 2 | cut -f 1` +echo "Length is $first_length seconds" + +trim_length=`echo "$first_length - $fade_length" | bc` + +# Get crossfade section from first file and optionally do the fade out +echo "Obtaining $fade_length seconds of fade out portion from $first_file..." +$SOX "$first_file" -e signed-integer -b 16 fadeout1.wav trim $trim_length + +# When user specifies "auto" try to guess if a fadeout is needed. +# "RMS amplitude" from the stat effect is effectively an average +# value of samples for the whole fade length file. If it seems +# quite then assume a fadeout has already been done. An RMS value +# of 0.1 was just obtained from trail and error. +if [ "$fade_first" == "auto" ]; then + RMS=`$SOX fadeout1.wav 2>&1 -n stat | grep RMS | grep amplitude | cut -d : -f 2 | cut -f 1` + should_fade=`echo "$RMS > 0.1" | bc` + if [ $should_fade == 0 ]; then + echo "Auto mode decided not to fadeout with RMS of $RMS" + fade_first_opts="" + else + echo "Auto mode will fadeout" + fi +fi + +$SOX fadeout1.wav fadeout2.wav $fade_first_opts + +# Get the crossfade section from the second file and optionally do the fade in +echo "Obtaining $fade_length seconds of fade in portion from $second_file..." +$SOX "$second_file" -e signed-integer -b 16 fadein1.wav trim 0 $fade_length + +# For auto, do similar thing as for fadeout. +if [ "$fade_second" == "auto" ]; then + RMS=`$SOX fadein1.wav 2>&1 -n stat | grep RMS | grep amplitude | cut -d : -f 2 | cut -f 1` + should_fade=`echo "$RMS > 0.1" | bc` + if [ $should_fade == 0 ]; then + echo "Auto mode decided not to fadein with RMS of $RMS" + fade_second_opts="" + else + echo "Auto mode will fadein" + fi +fi + +$SOX fadein1.wav fadein2.wav $fade_second_opts + +# Mix the crossfade files together at full volume +echo "Crossfading..." +$SOX -m -v 1.0 fadeout2.wav -v 1.0 fadein2.wav crossfade.wav + +echo "Trimming off crossfade sections from original files..." + +$SOX "$first_file" -e signed-integer -b 16 song1.wav trim 0 $trim_length +$SOX "$second_file" -e signed-integer -b 16 song2.wav trim $fade_length +$SOX song1.wav crossfade.wav song2.wav mix.wav + +echo -e "Removing temporary files...\n" +rm fadeout1.wav fadeout2.wav fadein1.wav fadein2.wav crossfade.wav song1.wav song2.wav +mins=`echo "$trim_length / 60" | bc` +secs=`echo "$trim_length % 60" | bc` +echo "The crossfade in mix.wav occurs at around $mins mins $secs secs" + diff --git a/bin/loop.sh b/bin/loop.sh new file mode 100755 index 0000000..beb1286 --- /dev/null +++ b/bin/loop.sh @@ -0,0 +1,20 @@ +rm fades.txt +rm files.txt +rm mix.sh + +ITER=0 +NEXT=1 +for i in `ls -v *.wav` +do +echo "[0$ITER][$NEXT]acrossfade=ns=5:o=1:c1=tri:c2=tri[0$NEXT];" >> fades.txt +printf '\-i %s ' $i >> files.txt +ITER=$(expr $ITER + 1) +NEXT=$(expr $NEXT + 1) +done + +printf "ffmpeg " >> mix.sh +cat files.txt >> mix.sh +printf " -filter_complex \"" >> mix.sh +cat fades.txt >> mix.sh +echo '\" out.wav' >> mix.sh + diff --git a/bin/xfade.sh b/bin/xfade.sh new file mode 100755 index 0000000..5e861e0 --- /dev/null +++ b/bin/xfade.sh @@ -0,0 +1,23 @@ +crossfade_dur=1 +i=0 +limit=10000 + +for file in `ls *.wav | sort -V` +do + i=$((i+1)) + if [ $i -eq $limit ] + then + break + fi + + if [ $i -eq 1 ] + then + cp $file mix.wav + else + # ../../crossfade_cat.sh $crossfade_dur mix.wav $file yes yes + echo $file + sox mix.wav "$file" out.wav splice $(soxi -D mix.wav),0.01 + mv out.wav mix.wav + fi +done + diff --git a/datasets/count_subdirs.sh b/datasets/count_subdirs.sh new file mode 100755 index 0000000..3999b3c --- /dev/null +++ b/datasets/count_subdirs.sh @@ -0,0 +1 @@ +find -maxdepth 1 -type d | sort | while read -r dir; do printf "%s:\t" "$dir"; find "$dir" -type f | wc -l; done diff --git a/datasets/generate.sh b/datasets/generate.sh new file mode 100755 index 0000000..335928c --- /dev/null +++ b/datasets/generate.sh @@ -0,0 +1,34 @@ +function process () { + echo "____________________________________________________" + echo "process $1" + name=$1 + in="${name}.wav" + out="s_${in}" + ./spread.sh $in $out 0.99 0.01 1.01 + ./split44k.sh $out 8 "44k_$name" + rm $out +} +function ease_process () { + echo "____________________________________________________" + echo "ease_process $1" + name=$1 + step=$2 + in="${name}.wav" + sout="o_${in}" + out="s_${in}" + sox -v 0.95 $in $sout + ./spread.sh $sout $out 0.999 $step 1.001 + ./split44k.sh $out 8 "44k_$name" + rm $sout + rm $out +} +#ease_process '' 0.0000 +ease_process 'blblbl' 0.00001515 +ease_process 'faty-scrub1' 0.0000285 +ease_process 'faty-medieval' 0.00003 +ease_process 'faty-crystals' 0.0000111 +ease_process 'faty-vocal1' 0.000013 +ease_process 'faty-vocal2' 0.000028145 +ease_process 'faty-scrub2' 0.00000466 +ease_process 'siren' 0.0000275 + diff --git a/datasets/split44k.sh b/datasets/split44k.sh new file mode 100755 index 0000000..4884af1 --- /dev/null +++ b/datasets/split44k.sh @@ -0,0 +1,24 @@ +#/bin/sh + +if [ "$#" -ne 3 ]; then + echo "Usage: $0 " + exit +fi + +fn=$1 +chunk_size=$2 +dataset_path=$3 + +converted=".temp2.wav" +rm -f $converted +ffmpeg -i $fn -ac 1 -ar 44100 $converted + +mkdir $dataset_path +length=$(ffprobe -i $converted -show_entries format=duration -v quiet -of csv="p=0") +end=$(echo "$length / $chunk_size - 1" | bc) +echo "splitting..." +for i in $(seq 0 $end); do + ffmpeg -hide_banner -loglevel error -ss $(($i * $chunk_size)) -t $chunk_size -i $converted "$dataset_path/$i.wav" +done +echo "done" +rm -f $converted diff --git a/datasets/spread.sh b/datasets/spread.sh new file mode 100755 index 0000000..bec1da3 --- /dev/null +++ b/datasets/spread.sh @@ -0,0 +1,25 @@ + +if [ "$#" -ne 5 ]; then + echo "Usage: $0 " + exit +fi + +FN_IN=$1 +FN_OUT=$2 +RATE=$3 +STEP=$4 +MAX=$5 + +ITER=0 +while true; do + if (( $(echo "$RATE > $MAX" | bc -l) )); then + break + fi + let ITER+=1 + RATE=`echo "$RATE+$STEP" | bc` + sox $FN_IN "tmp_$ITER.wav" speed $RATE +done + +sox tmp_* $FN_OUT +rm tmp_* + diff --git a/generate.py b/generate.py new file mode 100644 index 0000000..92a930f --- /dev/null +++ b/generate.py @@ -0,0 +1,365 @@ +# CometML needs to be imported first. +try: + import comet_ml +except ImportError: + pass + +from model import SampleRNN, Predictor +from optim import gradient_clipping +from nn import sequence_nll_loss_bits +from trainer import Trainer +from trainer.plugins import ( + TrainingLossMonitor, ValidationPlugin, AbsoluteTimeMonitor, SaverPlugin, + GeneratorPlugin, StatsPlugin +) +from dataset import FolderDataset, DataLoader + +import torch +from torch.utils.trainer.plugins import Logger + +from natsort import natsorted + +from functools import reduce +import os +import shutil +import sys +from glob import glob +import re +import argparse + + +default_params = { + # model parameters + 'n_rnn': 1, + 'dim': 1024, + 'learn_h0': True, + 'q_levels': 256, + 'seq_len': 1024, + 'weight_norm': True, + 'batch_size': 128, + 'val_frac': 0.1, + 'test_frac': 0.1, + + # training parameters + 'keep_old_checkpoints': False, + 'datasets_path': 'datasets', + 'results_path': 'results', + 'epoch_limit': 1000, + 'resume': True, + 'sample_rate': 16000, + 'n_samples': 1, + 'sample_length': 80000, + 'loss_smoothing': 0.99, + 'cuda': True, + 'comet_key': None +} + +tag_params = [ + 'exp', 'frame_sizes', 'n_rnn', 'dim', 'learn_h0', 'q_levels', 'seq_len', + 'batch_size', 'dataset', 'val_frac', 'test_frac' +] + +def param_to_string(value): + if isinstance(value, bool): + return 'T' if value else 'F' + elif isinstance(value, list): + return ','.join(map(param_to_string, value)) + else: + return str(value) + +def make_tag(params): + return '-'.join( + key + ':' + param_to_string(params[key]) + for key in tag_params + if key not in default_params or params[key] != default_params[key] + ) + +def setup_results_dir(params): + def ensure_dir_exists(path): + if not os.path.exists(path): + os.makedirs(path) + + tag = make_tag(params) + results_path = os.path.abspath(params['results_path']) + ensure_dir_exists(results_path) + results_path = os.path.join(results_path, tag) + if not os.path.exists(results_path): + os.makedirs(results_path) + elif not params['resume']: + shutil.rmtree(results_path) + os.makedirs(results_path) + + for subdir in ['checkpoints', 'samples']: + ensure_dir_exists(os.path.join(results_path, subdir)) + + return results_path + +def load_last_checkpoint(checkpoints_path): + checkpoints_pattern = os.path.join( + checkpoints_path, SaverPlugin.last_pattern.format('*', '*') + ) + checkpoint_paths = natsorted(glob(checkpoints_pattern)) + if len(checkpoint_paths) > 0: + checkpoint_path = checkpoint_paths[-1] + checkpoint_name = os.path.basename(checkpoint_path) + match = re.match( + SaverPlugin.last_pattern.format(r'(\d+)', r'(\d+)'), + checkpoint_name + ) + epoch = int(match.group(1)) + iteration = int(match.group(2)) + return (torch.load(checkpoint_path), epoch, iteration) + else: + return None + +def tee_stdout(log_path): + log_file = open(log_path, 'a', 1) + stdout = sys.stdout + + class Tee: + + def write(self, string): + log_file.write(string) + stdout.write(string) + + def flush(self): + log_file.flush() + stdout.flush() + + sys.stdout = Tee() + +def make_data_loader(overlap_len, params): + path = os.path.join(params['datasets_path'], params['dataset']) + def data_loader(split_from, split_to, eval): + dataset = FolderDataset( + path, overlap_len, params['q_levels'], split_from, split_to + ) + return DataLoader( + dataset, + batch_size=params['batch_size'], + seq_len=params['seq_len'], + overlap_len=overlap_len, + shuffle=(not eval), + drop_last=(not eval) + ) + return data_loader + +def init_comet(params, trainer): + if params['comet_key'] is not None: + from comet_ml import Experiment + from trainer.plugins import CometPlugin + experiment = Experiment(api_key=params['comet_key'], log_code=False) + hyperparams = { + name: param_to_string(params[name]) for name in tag_params + } + experiment.log_multiple_params(hyperparams) + trainer.register_plugin(CometPlugin( + experiment, [ + ('training_loss', 'epoch_mean'), + 'validation_loss', + 'test_loss' + ] + )) + +def main(exp, frame_sizes, dataset, **params): + params = dict( + default_params, + exp=exp, frame_sizes=frame_sizes, dataset=dataset, + **params + ) + + results_path = setup_results_dir(params) + tee_stdout(os.path.join(results_path, 'log')) + + model = SampleRNN( + frame_sizes=params['frame_sizes'], + n_rnn=params['n_rnn'], + dim=params['dim'], + learn_h0=params['learn_h0'], + q_levels=params['q_levels'], + weight_norm=params['weight_norm'] + ) + predictor = Predictor(model) + if params['cuda']: + model = model.cuda() + predictor = predictor.cuda() + + optimizer = gradient_clipping(torch.optim.Adam(predictor.parameters())) + + data_loader = make_data_loader(model.lookback, params) + test_split = 1 - params['test_frac'] + val_split = test_split - params['val_frac'] + + trainer = Trainer( + predictor, sequence_nll_loss_bits, optimizer, + data_loader(0, val_split, eval=False), + cuda=params['cuda'] + ) + + checkpoints_path = os.path.join(results_path, 'checkpoints') + checkpoint_data = load_last_checkpoint(checkpoints_path) + if checkpoint_data is not None: + (state_dict, epoch, iteration) = checkpoint_data + trainer.epochs = epoch + trainer.iterations = iteration + predictor.load_state_dict(state_dict) + print("epochs: {} iterations: {}".format(epoch, iteration)) + + """ + trainer.register_plugin(TrainingLossMonitor( + smoothing=params['loss_smoothing'] + )) + """ + trainer.register_plugin(ValidationPlugin( + data_loader(val_split, test_split, eval=True), + data_loader(test_split, 1, eval=True) + )) + trainer.register_plugin(AbsoluteTimeMonitor()) + """ + trainer.register_plugin(SaverPlugin( + checkpoints_path, params['keep_old_checkpoints'] + )) + """ + trainer.register_plugin(GeneratorPlugin( + os.path.join(results_path, 'samples'), params['n_samples'], + params['sample_length'], params['sample_rate'] + )) + """ + trainer.register_plugin( + Logger([ + 'training_loss', + 'validation_loss', + 'test_loss', + 'time' + ]) + ) + trainer.register_plugin(StatsPlugin( + results_path, + iteration_fields=[ + 'training_loss', + ('training_loss', 'running_avg'), + 'time' + ], + epoch_fields=[ + 'validation_loss', + 'test_loss', + 'time' + ], + plots={ + 'loss': { + 'x': 'iteration', + 'ys': [ + 'training_loss', + ('training_loss', 'running_avg'), + 'validation_loss', + 'test_loss', + ], + 'log_y': True + } + } + )) + init_comet(params, trainer) + """ + trainer.generate(int(params['epoch_limit'])) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + argument_default=argparse.SUPPRESS + ) + + def parse_bool(arg): + arg = arg.lower() + if 'true'.startswith(arg): + return True + elif 'false'.startswith(arg): + return False + else: + raise ValueError() + + parser.add_argument('--exp', required=True, help='experiment name') + parser.add_argument( + '--frame_sizes', nargs='+', type=int, required=True, + help='frame sizes in terms of the number of lower tier frames, \ + starting from the lowest RNN tier' + ) + parser.add_argument( + '--dataset', required=True, + help='dataset name - name of a directory in the datasets path \ + (settable by --datasets_path)' + ) + parser.add_argument( + '--n_rnn', type=int, help='number of RNN layers in each tier' + ) + parser.add_argument( + '--dim', type=int, help='number of neurons in every RNN and MLP layer' + ) + parser.add_argument( + '--learn_h0', type=parse_bool, + help='whether to learn the initial states of RNNs' + ) + parser.add_argument( + '--q_levels', type=int, + help='number of bins in quantization of audio samples' + ) + parser.add_argument( + '--seq_len', type=int, + help='how many samples to include in each truncated BPTT pass' + ) + parser.add_argument( + '--weight_norm', type=parse_bool, + help='whether to use weight normalization' + ) + parser.add_argument('--batch_size', type=int, help='batch size') + parser.add_argument( + '--val_frac', type=float, + help='fraction of data to go into the validation set' + ) + parser.add_argument( + '--test_frac', type=float, + help='fraction of data to go into the test set' + ) + parser.add_argument( + '--keep_old_checkpoints', type=parse_bool, + help='whether to keep checkpoints from past epochs' + ) + parser.add_argument( + '--datasets_path', help='path to the directory containing datasets' + ) + parser.add_argument( + '--results_path', help='path to the directory to save the results to' + ) + parser.add_argument('--epoch_limit', help='how many epochs to run') + parser.add_argument( + '--resume', type=parse_bool, default=True, + help='whether to resume training from the last checkpoint' + ) + parser.add_argument( + '--sample_rate', type=int, + help='sample rate of the training data and generated sound' + ) + parser.add_argument( + '--n_samples', type=int, + help='number of samples to generate in each epoch' + ) + parser.add_argument( + '--sample_length', type=int, + help='length of each generated sample (in samples)' + ) + parser.add_argument( + '--loss_smoothing', type=float, + help='smoothing parameter of the exponential moving average over \ + training loss, used in the log and in the loss plot' + ) + parser.add_argument( + '--cuda', type=parse_bool, + help='whether to use CUDA' + ) + parser.add_argument( + '--comet_key', help='comet.ml API key' + ) + + parser.set_defaults(**default_params) + + main(**vars(parser.parse_args())) diff --git a/glass_test2.sh b/glass_test2.sh new file mode 100644 index 0000000..072c451 --- /dev/null +++ b/glass_test2.sh @@ -0,0 +1,85 @@ +function runq () { + dataset=$1 + sample_rate=$2 + duration=$3 + let sample_length=$2*$3 + fs1=$4 + fs2=$5 + qs=$6 + + exp_name="space_q$qs" + + echo "__________________________________________" + echo "__________________________________________" + echo "__________________________________________" + echo "" + echo ">> running $exp_name" + echo "" + echo "__________________________________________" + echo "__________________________________________" + echo "__________________________________________" + + python train.py \ + --exp $exp_name --dataset $dataset \ + --frame_sizes $fs1 $fs2 \ + --n_rnn 2 --dim 1024 --q_levels 512 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit 6 \ + --resume True +} +function generateq () { + dataset=$1 + sample_rate=$2 + duration=$3 + let sample_length=$2*$3 + fs1=$4 + fs2=$5 + qs=$6 + + exp_name="space_q$qs" + + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo "" + echo ">> generating $exp_name" + echo "" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + + python generate.py \ + --exp $exp_name --dataset $dataset \ + --frame_sizes $fs1 $fs2 \ + --n_rnn 2 --dim 1024 --q_levels 512 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit 6 \ + --resume True +} + +#runq 44k_glass_space1 44100 5 8 2 32 +runq 44k_glass_space1 44100 5 8 2 1 +runq 44k_glass_space1 44100 5 8 2 2 +runq 44k_glass_space1 44100 5 8 2 4 +runq 44k_glass_space1 44100 5 8 2 8 +runq 44k_glass_space1 44100 5 8 2 16 +runq 44k_glass_space1 44100 5 8 2 32 +#runq 44k_glass_space1 44100 5 8 2 2048 + +#generateq 44k_glass_space1 44100 5 8 2 32 +#generateq 44k_glass_space1 44100 5 8 2 1 +#generateq 44k_glass_space1 44100 5 8 2 2 +#generateq 44k_glass_space1 44100 5 8 2 4 +#generateq 44k_glass_space1 44100 5 8 2 8 +#generateq 44k_glass_space1 44100 5 8 2 16 +#generateq 44k_glass_space1 44100 5 8 2 32 +#generateq 44k_glass_space1 44100 5 8 2 2048 + diff --git a/glass_test3.sh b/glass_test3.sh new file mode 100644 index 0000000..029cf58 --- /dev/null +++ b/glass_test3.sh @@ -0,0 +1,112 @@ +function runq () { + dataset=$1 + sample_rate=$2 + duration=$3 + let sample_length=$2*$3 + qs=$4 + + exp_name="space_qs$dim" + + echo "__________________________________________" + echo "__________________________________________" + echo "__________________________________________" + echo "" + echo ">> running $exp_name" + echo "" + echo "__________________________________________" + echo "__________________________________________" + echo "__________________________________________" + + python train.py \ + --exp $exp_name --dataset $dataset \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels $qs \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit 4 \ + --resume True +} + +function rundim () { + dataset=$1 + sample_rate=$2 + duration=$3 + let sample_length=$2*$3 + dim=$4 + + exp_name="space_dim$dim" + + echo "__________________________________________" + echo "__________________________________________" + echo "__________________________________________" + echo "" + echo ">> running $exp_name" + echo "" + echo "__________________________________________" + echo "__________________________________________" + echo "__________________________________________" + + python train.py \ + --exp $exp_name --dataset $dataset \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim $dim --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit 8 \ + --resume True +} + +function runseq () { + dataset=$1 + sample_rate=$2 + duration=$3 + let sample_length=$2*$3 + seq=$4 + + exp_name="space_seq$seq" + + echo "__________________________________________" + echo "__________________________________________" + echo "__________________________________________" + echo "" + echo ">> running $exp_name" + echo "" + echo "__________________________________________" + echo "__________________________________________" + echo "__________________________________________" + + python train.py \ + --exp $exp_name --dataset $dataset \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len $seq --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit 4 \ + --resume True +} + +#rundim 44k_glass_space1 44100 5 4096 +runseq 44k_glass_space1 44100 5 512 +rundim 44k_glass_space1 44100 5 512 +#rundim 44k_glass_space1 44100 5 256 +#rundim 44k_glass_space1 44100 5 128 +#rundim 44k_glass_space1 44100 5 64 +#rundim 44k_glass_space1 44100 5 32 +#rundim 44k_glass_space1 44100 5 16 +#rundim 44k_glass_space1 44100 5 8 +#rundim 44k_glass_space1 44100 5 4 +#rundim 44k_glass_space1 44100 5 2 +#rundim 44k_glass_space1 44100 5 1 + +runq 44k_glass_space1 44100 5 1024 +runq 44k_glass_space1 44100 5 128 + diff --git a/glass_tests.sh b/glass_tests.sh new file mode 100644 index 0000000..62d8932 --- /dev/null +++ b/glass_tests.sh @@ -0,0 +1,202 @@ +function run2 () { + dataset=$1 + sample_rate=$2 + duration=$3 + let sample_length=$2*$3 + fs1=$4 + fs2=$5 + + exp_name=space_2 + + echo "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@" + echo "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@" + echo "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@" + echo "" + echo ">> running $exp_name $4 $5" + echo "" + echo "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@" + echo "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@" + echo "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@" + python train.py \ + --exp $exp_name --dataset $dataset \ + --frame_sizes $fs1 $fs2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit 2 \ + --resume True +} +function generate2 () { + dataset=$1 + sample_rate=$2 + duration=$3 + let sample_length=$2*$3 + fs1=$4 + fs2=$5 + + exp_name=space_2 + + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo "" + echo ">> generating $exp_name $4 $5" + echo "" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + python generate.py \ + --exp $exp_name --dataset $dataset \ + --frame_sizes $fs1 $fs2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit 1 \ + --n_samples 6 \ + --resume True +} +function run3 () { + dataset=$1 + sample_rate=$2 + duration=$3 + let sample_length=$2*$3 + fs1=$4 + fs2=$5 + fs3=$6 + + exp_name=space_3 + + echo ">> running $exp_name $4 $5 $6" + python train.py \ + --exp $exp_name --dataset $dataset \ + --frame_sizes $fs1 $fs2 $fs3 \ + --n_rnn 3 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit 1 \ + --resume True +} + +function run4 () { + dataset=$1 + sample_rate=$2 + duration=$3 + let sample_length=$2*$3 + fs1=$4 + fs2=$5 + fs3=$6 + fs4=$7 + + exp_name=space_4 + + echo ">> running $exp_name $4 $5 $6 $7" + python train.py \ + --exp $exp_name --dataset $dataset \ + --frame_sizes $fs1 $fs2 $fs3 $fs4 \ + --n_rnn 4 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit 4 \ + --resume True +} +function run5 () { + dataset=$1 + sample_rate=$2 + duration=$3 + let sample_length=$2*$3 + fs1=$4 + fs2=$5 + fs3=$6 + fs4=$7 + fs5=$8 + + exp_name=space_5 + + echo ">> running $exp_name $4 $5 $6 $7 $8" + python train.py \ + --exp $exp_name --dataset $dataset \ + --frame_sizes $fs1 $fs2 $fs3 $fs4 $fs5 \ + --n_rnn 5 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit 4 \ + --resume True +} +function run6 () { + dataset=$1 + sample_rate=$2 + duration=$3 + let sample_length=$2*$3 + fs1=$4 + fs2=$5 + fs3=$6 + fs4=$7 + fs5=$8 + fs6=$9 + + exp_name=space_6 + + echo ">> running $exp_name $4 $5 $6 $7 $8 $9" + python train.py \ + --exp $exp_name --dataset $dataset \ + --frame_sizes $fs1 $fs2 $fs3 $fs4 $fs5 $fs6 \ + --n_rnn 5 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit 4 \ + --resume True +} + +run4 44k_glass_space1 44100 5 2 2 2 2 +run5 44k_glass_space1 44100 5 2 2 2 2 2 +run6 44k_glass_space1 44100 5 2 2 2 2 2 2 + +run4 44k_glass_space1 44100 5 8 2 2 2 +run5 44k_glass_space1 44100 5 8 2 2 2 2 +run6 44k_glass_space1 44100 5 8 2 2 2 2 2 +#generate2 44k_glass_space1 44100 5 8 2 +#run2 44k_glass_space1 44100 5 8 4 +#generate2 44k_glass_space1 44100 5 8 4 +#run2 44k_glass_space1 44100 5 8 1 +#generate2 44k_glass_space1 44100 5 8 1 +#run2 44k_glass_space1 44100 5 16 8 +#run2 44k_glass_space1 44100 5 16 4 +#run2 44k_glass_space1 44100 5 16 2 +#generate2 44k_glass_space1 44100 5 16 2 +#run2 44k_glass_space1 44100 5 16 1 +#generate2 44k_glass_space1 44100 5 16 1 + +#run2 44k_glass_space1 44100 5 4 2 +#run2 44k_glass_space1 44100 5 4 1 +#run2 44k_glass_space1 44100 5 2 1 + +#run3 44k_glass_space1 44100 5 8 2 1 +#run3 44k_glass_space1 44100 5 8 4 2 +#run3 44k_glass_space1 44100 5 16 8 2 +#run3 44k_glass_space1 44100 5 16 4 2 +#run3 44k_glass_space1 44100 5 16 2 1 + +#run2 44k_glass_space1 44100 5 6 2 +#run2 44k_glass_space1 44100 5 6 3 +#run3 44k_glass_space1 44100 5 5 3 2 +#run3 44k_glass_space1 44100 5 10 5 2 +#run3 44k_glass_space1 44100 5 8 5 2 + diff --git a/kick_test.sh b/kick_test.sh new file mode 100644 index 0000000..c3754a4 --- /dev/null +++ b/kick_test.sh @@ -0,0 +1,51 @@ +function generaterrr () { + exp_name=$1 + dataset=$2 + epoch_limit=$3 + sample_rate=$4 + duration=$5 + let sample_length=$4*$5 + qlev=$6 + + echo ">> generating $exp_name" + python generate.py \ + --exp $exp_name --dataset $dataset \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels $qlev \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit $epoch_limit \ + --resume True +} +function runrrr () { + exp_name=$1 + dataset=$2 + epoch_limit=$3 + sample_rate=$4 + duration=$5 + let sample_length=$4*$5 + qlev=$6 + + echo ">> running $exp_name" + python train.py \ + --exp $exp_name --dataset $dataset \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels $qlev \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit $epoch_limit \ + --resume True +} + +runrrr kiq256 kiq 1 44100 1 256 +generaterrr kiq256 kiq 6 44100 1 256 + +runrrr kiq512 kiq 1 44100 1 512 +generaterrr kiq512 kiq 10 44100 1 512 + diff --git a/mix.sh b/mix.sh new file mode 100755 index 0000000..d95c4ef --- /dev/null +++ b/mix.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 render_name" + exit +fi + +dir=$1 +name=$2 +now=`date +'%Y%m%d'` + +cd "$dir/samples" +mkdir "$name" +../../xfade.sh +cp mix.wav "../../../output/$name.wav" +mv *.wav "$name" +cd .. +vim log +cd ../.. + diff --git a/train.py b/train.py index e3061c9..b808cee 100644 --- a/train.py +++ b/train.py @@ -255,7 +255,7 @@ def main(exp, frame_sizes, dataset, **params): init_comet(params, trainer) - trainer.run(params['epoch_limit']) + trainer.run(int(params['epoch_limit'])) if __name__ == '__main__': diff --git a/train_basic_22k.sh b/train_basic_22k.sh new file mode 100644 index 0000000..5104034 --- /dev/null +++ b/train_basic_22k.sh @@ -0,0 +1,15 @@ +exp_name=$1 +dataset=$2 +epoch_limit=$3 +python train.py \ + --exp $exp_name --dataset $exp_name \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 64 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate 22050 \ + --sample_length 110250 \ + --keep_old_checkpoints False \ + --epoch_limit $epoch_limit \ + --resume True + diff --git a/train_basic_32k.sh b/train_basic_32k.sh new file mode 100644 index 0000000..c4d1319 --- /dev/null +++ b/train_basic_32k.sh @@ -0,0 +1,15 @@ +exp_name=$1 +dataset=$2 +epoch_limit=$3 +python train.py \ + --exp $exp_name --dataset $exp_name \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 64 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate 32000 \ + --sample_length 160000 \ + --keep_old_checkpoints False \ + --epoch_limit $epoch_limit \ + --resume True + diff --git a/train_basic_44k.sh b/train_basic_44k.sh new file mode 100644 index 0000000..1908f5d --- /dev/null +++ b/train_basic_44k.sh @@ -0,0 +1,15 @@ +exp_name=$1 +dataset=$2 +epoch_limit=$3 +python train.py \ + --exp $exp_name --dataset $exp_name \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 64 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate 44100 \ + --sample_length 220500 \ + --keep_old_checkpoints False \ + --epoch_limit $epoch_limit \ + --resume True + diff --git a/train_drums.sh b/train_drums.sh new file mode 100644 index 0000000..2157fc7 --- /dev/null +++ b/train_drums.sh @@ -0,0 +1,44 @@ +function generate () { + exp_name=$1 + epoch_limit=$2 + sample_rate=$3 + sample_length=$4 + + echo ">> generating $exp_name" + python generate.py \ + --exp $exp_name --dataset $exp_name \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit $epoch_limit \ + --resume True +} +function run () { + exp_name=$1 + epoch_limit=$2 + sample_rate=$3 + sample_length=$4 + + echo ">> running $exp_name" + python train.py \ + --exp $exp_name --dataset $exp_name \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --epoch_limit $epoch_limit \ + --resume True +} + +#generate 44k_jlin3 64 44100 4134 +generate 44k_jlin4 64 44100 4134 +generate 44k_clouds2 64 44100 4134 +generate 44k_glassmix 64 44100 4134 + diff --git a/train_test_generate.sh b/train_test_generate.sh new file mode 100644 index 0000000..af4f1c7 --- /dev/null +++ b/train_test_generate.sh @@ -0,0 +1,93 @@ +function generate () { + exp_name=$1 + n_samples=$2 + sample_rate=$3 + duration=$4 + let sample_length=$3*$4 + + echo "" + echo "###################################################" + echo "###################################################" + echo "###################################################" + echo "" + echo ">> generating $exp_name" + echo "" + echo "###################################################" + echo "###################################################" + echo "###################################################" + echo "" + python generate.py \ + --exp $exp_name --dataset $exp_name \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --n_samples $n_samples \ + --epoch_limit 1 \ + --resume True +} +function run () { + exp_name=$1 + epoch_limit=$2 + n_samples=$3 + sample_rate=$4 + duration=$5 + let sample_length=$4*$5 + + echo "" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo "" + echo ">> running $exp_name" + echo "" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo "" + python train.py \ + --exp $exp_name --dataset $exp_name \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --n_samples $n_samples \ + --keep_old_checkpoints False \ + --epoch_limit $epoch_limit \ + --resume True +} +function standard () { + dataset=$1 + run $1 6 6 44100 5 +} +function quick () { + dataset=$1 + run $1 4 6 44100 5 +} +function fast () { + dataset=$1 + run $1 1 6 44100 10 +} + +standard 44k_blblbl2 +standard 44k_faty-scrub2 +standard 44k_faty-vocal2 + +quick 44k_siren +quick 44k_whatifvocode +quick 44k_jlin-faty + +fast 44k_lipnoise +quick 44k_jlin-faty +quick 44k_jlin-faty + +run 44k_dances 8 6 44100 5 +run 44k_jlin4 4 3 44100 10 + +run 44k_sundae 4 3 44100 10 + diff --git a/trainer/__init__.py b/trainer/__init__.py index 7e2ea18..1f39506 100644 --- a/trainer/__init__.py +++ b/trainer/__init__.py @@ -56,6 +56,15 @@ class Trainer(object): self.train() self.call_plugins('epoch', self.epochs) + def generate(self, epochs=1): + for q in self.plugin_queues.values(): + heapq.heapify(q) + + for self.epochs in range(self.epochs + 1, self.epochs + epochs + 1): + # self.train() + self.call_plugins('update', self.iterations, self.model) + self.call_plugins('epoch', self.epochs) + def train(self): for (self.iterations, data) in \ enumerate(self.dataset, self.iterations + 1): diff --git a/trainer/plugins.py b/trainer/plugins.py index f8c299b..0126870 100644 --- a/trainer/plugins.py +++ b/trainer/plugins.py @@ -141,7 +141,7 @@ class SaverPlugin(Plugin): class GeneratorPlugin(Plugin): - pattern = 'ep{}-s{}.wav' + pattern = 'd-{}-ep{}-s{}.wav' def __init__(self, samples_path, n_samples, sample_length, sample_rate): super().__init__([(1, 'epoch')]) @@ -159,7 +159,7 @@ class GeneratorPlugin(Plugin): for i in range(self.n_samples): write_wav( os.path.join( - self.samples_path, self.pattern.format(epoch_index, i + 1) + self.samples_path, self.pattern.format(int(time.time()), epoch_index, i + 1) ), samples[i, :], sr=self.sample_rate, norm=True ) @@ -168,7 +168,7 @@ class GeneratorPlugin(Plugin): class StatsPlugin(Plugin): data_file_name = 'stats.pkl' - plot_pattern = '{}.svg' + plot_pattern = 'd-{}-{}.svg' def __init__(self, results_path, iteration_fields, epoch_fields, plots): super().__init__([(1, 'iteration'), (1, 'epoch')]) @@ -252,7 +252,7 @@ class StatsPlugin(Plugin): pyplot.legend() pyplot.savefig( - os.path.join(self.results_path, self.plot_pattern.format(name)) + os.path.join(self.results_path, self.plot_pattern.format(int(time.time()), name)) ) @staticmethod -- cgit v1.2.3-70-g09d2 From 261d7adff6049374edaa0167c02789dacf753e36 Mon Sep 17 00:00:00 2001 From: jules Date: Mon, 9 Apr 2018 00:11:18 +0200 Subject: epoch --- epoch.sh | 1 + 1 file changed, 1 insertion(+) create mode 100644 epoch.sh diff --git a/epoch.sh b/epoch.sh new file mode 100644 index 0000000..bbd8668 --- /dev/null +++ b/epoch.sh @@ -0,0 +1 @@ +ls -lat results/*/checkpoints/ep* | head -n 20 -- cgit v1.2.3-70-g09d2 From a7256287f599dbaaffb107df0b8b4ea4a2b1f2b1 Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Mon, 9 Apr 2018 00:13:27 +0200 Subject: dataset script --- datasets/dataset.pl | 67 ++++++++++++++++++++++++++++++++++++ datasets/spread.sh | 8 +++++ run.sh | 88 +++++++++++++++++++++++++++++++++++++++++++++++ train_test_generate.sh | 93 -------------------------------------------------- 4 files changed, 163 insertions(+), 93 deletions(-) create mode 100755 datasets/dataset.pl create mode 100644 run.sh delete mode 100644 train_test_generate.sh diff --git a/datasets/dataset.pl b/datasets/dataset.pl new file mode 100755 index 0000000..4e3b9a9 --- /dev/null +++ b/datasets/dataset.pl @@ -0,0 +1,67 @@ +#!/usr/bin/perl + +use strict; +use Getopt::Std; + +our $opt_c; +getopts('c'); + +my $fmt = < " @@ -11,15 +12,22 @@ STEP=$4 MAX=$5 ITER=0 + while true; do if (( $(echo "$RATE > $MAX" | bc -l) )); then break fi let ITER+=1 RATE=`echo "$RATE+$STEP" | bc` + if ((ITER % 20 != 0)); then + echo "${ITER}... ${RATE}" + sleep 1 + fi sox $FN_IN "tmp_$ITER.wav" speed $RATE done +echo "made $ITER copies" + sox tmp_* $FN_OUT rm tmp_* diff --git a/run.sh b/run.sh new file mode 100644 index 0000000..3237eb1 --- /dev/null +++ b/run.sh @@ -0,0 +1,88 @@ +function generate () { + exp_name=$1 + n_samples=$2 + sample_rate=$3 + duration=$4 + let sample_length=$3*$4 + + echo "" + echo "###################################################" + echo "###################################################" + echo "###################################################" + echo "" + echo ">> generating $exp_name" + echo "" + echo "###################################################" + echo "###################################################" + echo "###################################################" + echo "" + python generate.py \ + --exp $exp_name --dataset $exp_name \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --n_samples $n_samples \ + --epoch_limit 1 \ + --resume True +} +function run () { + exp_name=$1 + epoch_limit=$2 + n_samples=$3 + sample_rate=$4 + duration=$5 + let sample_length=$4*$5 + + echo "" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo "" + echo ">> running $exp_name" + echo "" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo "" + python train.py \ + --exp $exp_name --dataset $exp_name \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --n_samples $n_samples \ + --keep_old_checkpoints False \ + --epoch_limit $epoch_limit \ + --resume True +} +function standard () { + dataset=$1 + run $1 6 6 44100 5 +} +function quick () { + dataset=$1 + run $1 4 6 44100 5 +} +function fast () { + dataset=$1 + run $1 1 6 44100 10 +} + +quick 01_snapping_clean +quick 01_snapping_compress +quick 01_snapping_highpass + +quick 01_snapping_clean +quick 01_snapping_compress +quick 01_snapping_highpass + +quick 01_snapping_clean +quick 01_snapping_compress +quick 01_snapping_highpass + diff --git a/train_test_generate.sh b/train_test_generate.sh deleted file mode 100644 index af4f1c7..0000000 --- a/train_test_generate.sh +++ /dev/null @@ -1,93 +0,0 @@ -function generate () { - exp_name=$1 - n_samples=$2 - sample_rate=$3 - duration=$4 - let sample_length=$3*$4 - - echo "" - echo "###################################################" - echo "###################################################" - echo "###################################################" - echo "" - echo ">> generating $exp_name" - echo "" - echo "###################################################" - echo "###################################################" - echo "###################################################" - echo "" - python generate.py \ - --exp $exp_name --dataset $exp_name \ - --frame_sizes 8 2 \ - --n_rnn 2 --dim 1024 --q_levels 256 \ - --seq_len 1024 --batch_size 128 \ - --val_frac 0.1 --test_frac 0.1 \ - --sample_rate $sample_rate \ - --sample_length $sample_length \ - --keep_old_checkpoints False \ - --n_samples $n_samples \ - --epoch_limit 1 \ - --resume True -} -function run () { - exp_name=$1 - epoch_limit=$2 - n_samples=$3 - sample_rate=$4 - duration=$5 - let sample_length=$4*$5 - - echo "" - echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" - echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" - echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" - echo "" - echo ">> running $exp_name" - echo "" - echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" - echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" - echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" - echo "" - python train.py \ - --exp $exp_name --dataset $exp_name \ - --frame_sizes 8 2 \ - --n_rnn 2 --dim 1024 --q_levels 256 \ - --seq_len 1024 --batch_size 128 \ - --val_frac 0.1 --test_frac 0.1 \ - --sample_rate $sample_rate \ - --sample_length $sample_length \ - --n_samples $n_samples \ - --keep_old_checkpoints False \ - --epoch_limit $epoch_limit \ - --resume True -} -function standard () { - dataset=$1 - run $1 6 6 44100 5 -} -function quick () { - dataset=$1 - run $1 4 6 44100 5 -} -function fast () { - dataset=$1 - run $1 1 6 44100 10 -} - -standard 44k_blblbl2 -standard 44k_faty-scrub2 -standard 44k_faty-vocal2 - -quick 44k_siren -quick 44k_whatifvocode -quick 44k_jlin-faty - -fast 44k_lipnoise -quick 44k_jlin-faty -quick 44k_jlin-faty - -run 44k_dances 8 6 44100 5 -run 44k_jlin4 4 3 44100 10 - -run 44k_sundae 4 3 44100 10 - -- cgit v1.2.3-70-g09d2 From b4f99b98d3165db19320ced0a595ddb881113cba Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Mon, 9 Apr 2018 07:27:52 +0200 Subject: perms --- best-epoch.sh | 0 glass_test2.sh | 0 glass_test3.sh | 0 glass_tests.sh | 0 kick_test.sh | 0 run.sh | 0 train_basic_22k.sh | 0 train_basic_32k.sh | 0 train_basic_44k.sh | 0 train_drums.sh | 0 10 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 best-epoch.sh mode change 100644 => 100755 glass_test2.sh mode change 100644 => 100755 glass_test3.sh mode change 100644 => 100755 glass_tests.sh mode change 100644 => 100755 kick_test.sh mode change 100644 => 100755 run.sh mode change 100644 => 100755 train_basic_22k.sh mode change 100644 => 100755 train_basic_32k.sh mode change 100644 => 100755 train_basic_44k.sh mode change 100644 => 100755 train_drums.sh diff --git a/best-epoch.sh b/best-epoch.sh old mode 100644 new mode 100755 diff --git a/glass_test2.sh b/glass_test2.sh old mode 100644 new mode 100755 diff --git a/glass_test3.sh b/glass_test3.sh old mode 100644 new mode 100755 diff --git a/glass_tests.sh b/glass_tests.sh old mode 100644 new mode 100755 diff --git a/kick_test.sh b/kick_test.sh old mode 100644 new mode 100755 diff --git a/run.sh b/run.sh old mode 100644 new mode 100755 diff --git a/train_basic_22k.sh b/train_basic_22k.sh old mode 100644 new mode 100755 diff --git a/train_basic_32k.sh b/train_basic_32k.sh old mode 100644 new mode 100755 diff --git a/train_basic_44k.sh b/train_basic_44k.sh old mode 100644 new mode 100755 diff --git a/train_drums.sh b/train_drums.sh old mode 100644 new mode 100755 -- cgit v1.2.3-70-g09d2 From 6fe185e50393f57aa7125bb83866aeb96cb88c92 Mon Sep 17 00:00:00 2001 From: jules Date: Mon, 9 Apr 2018 07:39:22 +0200 Subject: including results scripts --- .gitignore | 5 +- results/crossfade_cat.sh | 129 +++++++++++++++++++++++++++++++++++++++++++++++ results/loop.sh | 20 ++++++++ results/xfade.sh | 25 +++++++++ 4 files changed, 178 insertions(+), 1 deletion(-) create mode 100755 results/crossfade_cat.sh create mode 100755 results/loop.sh create mode 100755 results/xfade.sh diff --git a/.gitignore b/.gitignore index 3bb05ff..c14acc1 100644 --- a/.gitignore +++ b/.gitignore @@ -98,5 +98,8 @@ ENV/ *.aif *.aiff -results/ +results/exp* +!results/*.sh + +run_here.sh diff --git a/results/crossfade_cat.sh b/results/crossfade_cat.sh new file mode 100755 index 0000000..13a8e90 --- /dev/null +++ b/results/crossfade_cat.sh @@ -0,0 +1,129 @@ +#!/bin/bash +# +# crossfade_cat.sh +# +# Concatenates two files together with a crossfade of $1 seconds. +# Filenames are specified as $2 and $3. +# +# $4 is optional and specifies if a fadeout should be performed on +# first file. +# $5 is optional and specifies if a fadein should be performed on +# second file. +# +# Example: crossfade_cat.sh 10 infile1.wav infile2.wav auto auto +# +# By default, the script attempts to guess if the audio files +# already have a fadein/out on them or if they just have really +# low volumes that won't cause clipping when mixxing. If this +# is not detected then the script will perform a fade in/out to +# prevent clipping. +# +# The user may specify "yes" or "no" to force the fade in/out +# to occur. They can also specify "auto" which is the default. +# +# Crossfaded file is created as "mix.wav". +# +# Original script from Kester Clegg. Mods by Chris Bagwell to show +# more examples of sox features. +# + +SOX=sox +SOXI=soxi + +if [ "$3" == "" ]; then + echo "Usage: $0 crossfade_seconds first_file second_file [ fadeout ] [ fadein ]" + echo + echo "If a fadeout or fadein is not desired then specify \"no\" for that option. \"yes\" will force a fade and \"auto\" will try to detect if a fade should occur." + echo + echo "Example: $0 10 infile1.wav infile2.wav auto auto" + exit 1 +fi + +fade_length=$1 +first_file=$2 +second_file=$3 + +fade_first="auto" +if [ "$4" != "" ]; then + fade_first=$4 +fi + +fade_second="auto" +if [ "$5" != "" ]; then + fade_second=$5 +fi + +fade_first_opts= +if [ "$fade_first" != "no" ]; then + fade_first_opts="fade t 0 0:0:$fade_length 0:0:$fade_length" +fi + +fade_second_opts= +if [ "$fade_second" != "no" ]; then + fade_second_opts="fade t 0:0:$fade_length" +fi + +echo "crossfade and concatenate files" +echo +echo "Finding length of $first_file..." +first_length=`$SOX "$first_file" 2>&1 -n stat | grep Length | cut -d : -f 2 | cut -f 1` +echo "Length is $first_length seconds" + +trim_length=`echo "$first_length - $fade_length" | bc` + +# Get crossfade section from first file and optionally do the fade out +echo "Obtaining $fade_length seconds of fade out portion from $first_file..." +$SOX "$first_file" -e signed-integer -b 16 fadeout1.wav trim $trim_length + +# When user specifies "auto" try to guess if a fadeout is needed. +# "RMS amplitude" from the stat effect is effectively an average +# value of samples for the whole fade length file. If it seems +# quite then assume a fadeout has already been done. An RMS value +# of 0.1 was just obtained from trail and error. +if [ "$fade_first" == "auto" ]; then + RMS=`$SOX fadeout1.wav 2>&1 -n stat | grep RMS | grep amplitude | cut -d : -f 2 | cut -f 1` + should_fade=`echo "$RMS > 0.1" | bc` + if [ $should_fade == 0 ]; then + echo "Auto mode decided not to fadeout with RMS of $RMS" + fade_first_opts="" + else + echo "Auto mode will fadeout" + fi +fi + +$SOX fadeout1.wav fadeout2.wav $fade_first_opts + +# Get the crossfade section from the second file and optionally do the fade in +echo "Obtaining $fade_length seconds of fade in portion from $second_file..." +$SOX "$second_file" -e signed-integer -b 16 fadein1.wav trim 0 $fade_length + +# For auto, do similar thing as for fadeout. +if [ "$fade_second" == "auto" ]; then + RMS=`$SOX fadein1.wav 2>&1 -n stat | grep RMS | grep amplitude | cut -d : -f 2 | cut -f 1` + should_fade=`echo "$RMS > 0.1" | bc` + if [ $should_fade == 0 ]; then + echo "Auto mode decided not to fadein with RMS of $RMS" + fade_second_opts="" + else + echo "Auto mode will fadein" + fi +fi + +$SOX fadein1.wav fadein2.wav $fade_second_opts + +# Mix the crossfade files together at full volume +echo "Crossfading..." +$SOX -m -v 1.0 fadeout2.wav -v 1.0 fadein2.wav crossfade.wav + +echo "Trimming off crossfade sections from original files..." + +$SOX "$first_file" -e signed-integer -b 16 song1.wav trim 0 $trim_length +$SOX "$second_file" -e signed-integer -b 16 song2.wav trim $fade_length +$SOX song1.wav crossfade.wav song2.wav mix.wav + +echo -e "Removing temporary files...\n" +rm fadeout1.wav fadeout2.wav fadein1.wav fadein2.wav crossfade.wav song1.wav song2.wav +mins=`echo "$trim_length / 60" | bc` +secs=`echo "$trim_length % 60" | bc` +echo "The crossfade in mix.wav occurs at around $mins mins $secs secs" + diff --git a/results/loop.sh b/results/loop.sh new file mode 100755 index 0000000..beb1286 --- /dev/null +++ b/results/loop.sh @@ -0,0 +1,20 @@ +rm fades.txt +rm files.txt +rm mix.sh + +ITER=0 +NEXT=1 +for i in `ls -v *.wav` +do +echo "[0$ITER][$NEXT]acrossfade=ns=5:o=1:c1=tri:c2=tri[0$NEXT];" >> fades.txt +printf '\-i %s ' $i >> files.txt +ITER=$(expr $ITER + 1) +NEXT=$(expr $NEXT + 1) +done + +printf "ffmpeg " >> mix.sh +cat files.txt >> mix.sh +printf " -filter_complex \"" >> mix.sh +cat fades.txt >> mix.sh +echo '\" out.wav' >> mix.sh + diff --git a/results/xfade.sh b/results/xfade.sh new file mode 100755 index 0000000..9fc4d39 --- /dev/null +++ b/results/xfade.sh @@ -0,0 +1,25 @@ +crossfade_dur=1 +i=0 +limit=10000 + +for file in `ls *.wav | sort -V` +do + i=$((i+1)) + if [ $i -eq $limit ] + then + break + fi + + if [ $i -eq 1 ] + then + cp $file mix.wav + else + # ../../crossfade_cat.sh $crossfade_dur mix.wav $file yes yes + echo $file + sox $file tmp.wav highpass 10 + sox mix.wav tmp.wav out.wav splice $(soxi -D mix.wav),0.01 + mv out.wav mix.wav + rm tmp.wav + fi +done + -- cgit v1.2.3-70-g09d2 From aeab66dae9711de2baa4a9fe7098c09be1a6107c Mon Sep 17 00:00:00 2001 From: jules Date: Mon, 9 Apr 2018 07:45:11 +0200 Subject: mixdown highpass/normalize --- mix.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mix.sh b/mix.sh index d95c4ef..592016e 100755 --- a/mix.sh +++ b/mix.sh @@ -12,7 +12,8 @@ now=`date +'%Y%m%d'` cd "$dir/samples" mkdir "$name" ../../xfade.sh -cp mix.wav "../../../output/$name.wav" +sox mix.wav "../../../output/$name.wav" norm +rm mix.wav mv *.wav "$name" cd .. vim log -- cgit v1.2.3-70-g09d2 From 56b66750e02030446679f9da3b0e249c6e32e7c6 Mon Sep 17 00:00:00 2001 From: jules Date: Mon, 9 Apr 2018 08:02:16 +0200 Subject: compress to mp3 --- mix.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mix.sh b/mix.sh index 592016e..97792ad 100755 --- a/mix.sh +++ b/mix.sh @@ -12,8 +12,9 @@ now=`date +'%Y%m%d'` cd "$dir/samples" mkdir "$name" ../../xfade.sh -sox mix.wav "../../../output/$name.wav" norm -rm mix.wav +sox mix.wav norm.wav norm +lame -V 0 norm.wav "../../../output/$name.wav" +rm mix.wav norm.wav mv *.wav "$name" cd .. vim log -- cgit v1.2.3-70-g09d2 From e604b502db345110ea9b2780682c2baaeb7a73c9 Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Mon, 9 Apr 2018 08:44:40 +0200 Subject: node stuff --- .gitignore | 3 + package-lock.json | 391 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ package.json | 21 +++ upload.js | 38 ++++++ 4 files changed, 453 insertions(+) create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 upload.js diff --git a/.gitignore b/.gitignore index c14acc1..836c30d 100644 --- a/.gitignore +++ b/.gitignore @@ -103,3 +103,6 @@ results/exp* run_here.sh +.env + +node_modules/ diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..cefb77b --- /dev/null +++ b/package-lock.json @@ -0,0 +1,391 @@ +{ + "name": "samplernn", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "ajv": { + "version": "5.5.2", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz", + "integrity": "sha1-c7Xuyj+rZT49P5Qis0GtQiBdyWU=", + "requires": { + "co": "4.6.0", + "fast-deep-equal": "1.1.0", + "fast-json-stable-stringify": "2.0.0", + "json-schema-traverse": "0.3.1" + } + }, + "asn1": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz", + "integrity": "sha1-2sh4dxPJlmhJ/IGAd36+nB3fO4Y=" + }, + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" + }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" + }, + "aws-sign2": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" + }, + "aws4": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.7.0.tgz", + "integrity": "sha512-32NDda82rhwD9/JBCCkB+MRYDp0oSvlo2IL6rQWA10PQi7tDUM3eqMSltXmY+Oyl/7N3P3qNtAlv7X0d9bI28w==" + }, + "bcrypt-pbkdf": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz", + "integrity": "sha1-Y7xdy2EzG5K8Bf1SiVPDNGKgb40=", + "optional": true, + "requires": { + "tweetnacl": "0.14.5" + } + }, + "boom": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/boom/-/boom-4.3.1.tgz", + "integrity": "sha1-T4owBctKfjiJ90kDD9JbluAdLjE=", + "requires": { + "hoek": "4.2.1" + } + }, + "caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" + }, + "co": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", + "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=" + }, + "combined-stream": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.6.tgz", + "integrity": "sha1-cj599ugBrFYTETp+RFqbactjKBg=", + "requires": { + "delayed-stream": "1.0.0" + } + }, + "core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, + "cryptiles": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-3.1.2.tgz", + "integrity": "sha1-qJ+7Ig9c4l7FboxKqKT9e1sNKf4=", + "requires": { + "boom": "5.2.0" + }, + "dependencies": { + "boom": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/boom/-/boom-5.2.0.tgz", + "integrity": "sha512-Z5BTk6ZRe4tXXQlkqftmsAUANpXmuwlsF5Oov8ThoMbQRzdGTA1ngYRW160GexgOgjsFOKJz0LYhoNi+2AMBUw==", + "requires": { + "hoek": "4.2.1" + } + } + } + }, + "dashdash": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", + "requires": { + "assert-plus": "1.0.0" + } + }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" + }, + "dotenv": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-5.0.1.tgz", + "integrity": "sha512-4As8uPrjfwb7VXC+WnLCbXK7y+Ueb2B3zgNCePYfhxS1PYeaO1YTeplffTEcbfLhvFNGLAz90VvJs9yomG7bow==" + }, + "ecc-jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz", + "integrity": "sha1-D8c6ntXw1Tw4GTOYUj735UN3dQU=", + "optional": true, + "requires": { + "jsbn": "0.1.1" + } + }, + "extend": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.1.tgz", + "integrity": "sha1-p1Xqe8Gt/MWjHOfnYtuq3F5jZEQ=" + }, + "extsprintf": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" + }, + "fast-deep-equal": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz", + "integrity": "sha1-wFNHeBfIa1HaqFPIHgWbcz0CNhQ=" + }, + "fast-json-stable-stringify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", + "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" + }, + "forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" + }, + "form-data": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.2.tgz", + "integrity": "sha1-SXBJi+YEwgwAXU9cI67NIda0kJk=", + "requires": { + "asynckit": "0.4.0", + "combined-stream": "1.0.6", + "mime-types": "2.1.18" + } + }, + "getpass": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", + "requires": { + "assert-plus": "1.0.0" + } + }, + "har-schema": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" + }, + "har-validator": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.0.3.tgz", + "integrity": "sha1-ukAsJmGU8VlW7xXg/PJCmT9qff0=", + "requires": { + "ajv": "5.5.2", + "har-schema": "2.0.0" + } + }, + "hawk": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/hawk/-/hawk-6.0.2.tgz", + "integrity": "sha512-miowhl2+U7Qle4vdLqDdPt9m09K6yZhkLDTWGoUiUzrQCn+mHHSmfJgAyGaLRZbPmTqfFFjRV1QWCW0VWUJBbQ==", + "requires": { + "boom": "4.3.1", + "cryptiles": "3.1.2", + "hoek": "4.2.1", + "sntp": "2.1.0" + } + }, + "hoek": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/hoek/-/hoek-4.2.1.tgz", + "integrity": "sha512-QLg82fGkfnJ/4iy1xZ81/9SIJiq1NGFUMGs6ParyjBZr6jW2Ufj/snDqTHixNlHdPNwN2RLVD0Pi3igeK9+JfA==" + }, + "http-signature": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", + "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", + "requires": { + "assert-plus": "1.0.0", + "jsprim": "1.4.1", + "sshpk": "1.14.1" + } + }, + "is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" + }, + "isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" + }, + "jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=", + "optional": true + }, + "json-schema": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" + }, + "json-schema-traverse": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz", + "integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A=" + }, + "json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" + }, + "jsprim": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", + "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "requires": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.2.3", + "verror": "1.10.0" + } + }, + "mime-db": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.33.0.tgz", + "integrity": "sha512-BHJ/EKruNIqJf/QahvxwQZXKygOQ256myeN/Ew+THcAa5q+PjyTTMMeNQC4DZw5AwfvelsUrA6B67NKMqXDbzQ==" + }, + "mime-types": { + "version": "2.1.18", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.18.tgz", + "integrity": "sha512-lc/aahn+t4/SWV/qcmumYjymLsWfN3ELhpmVuUFjgsORruuZPVSwAQryq+HHGvO/SI2KVX26bx+En+zhM8g8hQ==", + "requires": { + "mime-db": "1.33.0" + } + }, + "minimist": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" + }, + "oauth-sign": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz", + "integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM=" + }, + "performance-now": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" + }, + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + }, + "qs": { + "version": "6.5.1", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz", + "integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A==" + }, + "request": { + "version": "2.85.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.85.0.tgz", + "integrity": "sha512-8H7Ehijd4js+s6wuVPLjwORxD4zeuyjYugprdOXlPSqaApmL/QOy+EB/beICHVCHkGMKNh5rvihb5ov+IDw4mg==", + "requires": { + "aws-sign2": "0.7.0", + "aws4": "1.7.0", + "caseless": "0.12.0", + "combined-stream": "1.0.6", + "extend": "3.0.1", + "forever-agent": "0.6.1", + "form-data": "2.3.2", + "har-validator": "5.0.3", + "hawk": "6.0.2", + "http-signature": "1.2.0", + "is-typedarray": "1.0.0", + "isstream": "0.1.2", + "json-stringify-safe": "5.0.1", + "mime-types": "2.1.18", + "oauth-sign": "0.8.2", + "performance-now": "2.1.0", + "qs": "6.5.1", + "safe-buffer": "5.1.1", + "stringstream": "0.0.5", + "tough-cookie": "2.3.4", + "tunnel-agent": "0.6.0", + "uuid": "3.2.1" + } + }, + "safe-buffer": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", + "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==" + }, + "sntp": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/sntp/-/sntp-2.1.0.tgz", + "integrity": "sha512-FL1b58BDrqS3A11lJ0zEdnJ3UOKqVxawAkF3k7F0CVN7VQ34aZrV+G8BZ1WC9ZL7NyrwsW0oviwsWDgRuVYtJg==", + "requires": { + "hoek": "4.2.1" + } + }, + "sshpk": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.14.1.tgz", + "integrity": "sha1-Ew9Zde3a2WPx1W+SuaxsUfqfg+s=", + "requires": { + "asn1": "0.2.3", + "assert-plus": "1.0.0", + "bcrypt-pbkdf": "1.0.1", + "dashdash": "1.14.1", + "ecc-jsbn": "0.1.1", + "getpass": "0.1.7", + "jsbn": "0.1.1", + "tweetnacl": "0.14.5" + } + }, + "stringstream": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz", + "integrity": "sha1-TkhM1N5aC7vuGORjB3EKioFiGHg=" + }, + "tough-cookie": { + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.4.tgz", + "integrity": "sha512-TZ6TTfI5NtZnuyy/Kecv+CnoROnyXn2DN97LontgQpCwsX2XyLYCC0ENhYkehSOwAp8rTQKc/NUIF7BkQ5rKLA==", + "requires": { + "punycode": "1.4.1" + } + }, + "tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "requires": { + "safe-buffer": "5.1.1" + } + }, + "tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", + "optional": true + }, + "uuid": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.2.1.tgz", + "integrity": "sha512-jZnMwlb9Iku/O3smGWvZhauCf6cvvpKi4BKRiliS3cxnI+Gz9j5MEpTz2UFuXiKPJocb7gnsLHwiS05ige5BEA==" + }, + "verror": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "requires": { + "assert-plus": "1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "1.3.0" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..838862f --- /dev/null +++ b/package.json @@ -0,0 +1,21 @@ +{ + "name": "samplernn", + "version": "1.0.0", + "description": "samplernn", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git@asdf.us:samplernn.git" + }, + "author": "jules ", + "license": "UNLICENSED", + "private": true, + "dependencies": { + "dotenv": "^5.0.1", + "minimist": "^1.2.0", + "request": "^2.85.0" + } +} diff --git a/upload.js b/upload.js new file mode 100644 index 0000000..1cdcba8 --- /dev/null +++ b/upload.js @@ -0,0 +1,38 @@ +#!`which node` + +require('dotenv').load(); +var request = require('request') +var path = require('path') +var fs = require('fs') +var argv = require('minimist')(process.argv.slice(2)); + +var request = request.defaults({jar: true}) + +var j = request.jar() +var cookie = request.cookie('bucky.sid=' + process.env.COOKIE) +j.setCookie(cookie, 'https://bucky.asdf.us/'); +var request = request.defaults({jar:j}) + +var id = process.env.THREAD_ID + +console.log(argv) + +if (!argv['_'] || ! argv['_'].length) { + console.error('not enough args!') + process.exit() +} + +var r = request.post('https://bucky.asdf.us/api/thread/' + id + '/comment', function (err, res, body) { + if (err) { + return console.error('upload failed:', err); + } + // console.log(err, res, body) + console.log('Upload successful'); +}) + +var form = r.form() +form.append('comment', '') +form.append('csrf', 'csrf') +argv['_'].forEach(fn => { + form.append('files', fs.createReadStream(path.join(__dirname, fn))) +}) -- cgit v1.2.3-70-g09d2 From e4e8f869069e773a1bd00dda4da68df3dcbcc75b Mon Sep 17 00:00:00 2001 From: jules Date: Mon, 9 Apr 2018 08:50:34 +0200 Subject: mix --- mix.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mix.sh b/mix.sh index 97792ad..4fddfdf 100755 --- a/mix.sh +++ b/mix.sh @@ -13,10 +13,12 @@ cd "$dir/samples" mkdir "$name" ../../xfade.sh sox mix.wav norm.wav norm -lame -V 0 norm.wav "../../../output/$name.wav" +lame -V 0 norm.wav "../../../output/$name.mp3" rm mix.wav norm.wav mv *.wav "$name" cd .. vim log cd ../.. +node upload.js "./output/$name.mp3" + -- cgit v1.2.3-70-g09d2 From c92ef66869e11f42164d1f0ecca4b004fb89a572 Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Mon, 9 Apr 2018 08:54:25 +0200 Subject: grep --- mix.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.sh b/mix.sh index 4fddfdf..c212cfa 100755 --- a/mix.sh +++ b/mix.sh @@ -17,7 +17,7 @@ lame -V 0 norm.wav "../../../output/$name.mp3" rm mix.wav norm.wav mv *.wav "$name" cd .. -vim log +grep valid log cd ../.. node upload.js "./output/$name.mp3" -- cgit v1.2.3-70-g09d2 From 4154b89a772bbb8bb61d69b823485a74209b9b23 Mon Sep 17 00:00:00 2001 From: jules Date: Tue, 10 Apr 2018 00:40:42 +0200 Subject: dataset only process audio --- datasets/dataset.pl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/datasets/dataset.pl b/datasets/dataset.pl index 4e3b9a9..8b77af9 100755 --- a/datasets/dataset.pl +++ b/datasets/dataset.pl @@ -20,6 +20,10 @@ FMT sub process($) { my $filename = shift or die "Usage: $0 [...filenames]\n"; my ($name, $ext) = split(/\./, $filename, 2); + if ($ext !~ /(wav|mp3|aiff?|flac)/) { + print "not a valid file extension: $ext\n"; + return; + } my $soxi = `soxi $filename`; my @lines = split("\n", $soxi); -- cgit v1.2.3-70-g09d2 From 630c245ad0d42ab4a83198c4c04f78feaa1a0428 Mon Sep 17 00:00:00 2001 From: jules Date: Tue, 10 Apr 2018 00:41:29 +0200 Subject: ignore runhere scripts --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 836c30d..8ab7957 100644 --- a/.gitignore +++ b/.gitignore @@ -101,8 +101,9 @@ ENV/ results/exp* !results/*.sh -run_here.sh +run_here* .env node_modules/ + -- cgit v1.2.3-70-g09d2 From 001c18201e06b0740b8aff7d7a613bb1dcb93553 Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Tue, 10 Apr 2018 02:02:13 +0200 Subject: script to list latest files --- latest.pl | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ mix.sh | 2 ++ 2 files changed, 51 insertions(+) create mode 100644 latest.pl diff --git a/latest.pl b/latest.pl new file mode 100644 index 0000000..b20ca1c --- /dev/null +++ b/latest.pl @@ -0,0 +1,49 @@ +#!/usr/bin/perl + +use strict; +use Getopt::Std; + +our $opt_l; +getopts('l'); + +# my $s = "results/exp:03_responses-frame_sizes:8,2-n_rnn:2-dataset:03_responses/checkpoints/best-ep1-it1035"; +sub process($){ + my ($dir) = @_; + my @exp = split(":", $dir); + my $name = $exp[4]; + my $ep; + my $path = "results/" . $dir . "/"; + my $chex = $path . "checkpoints/"; + return 0 unless -e $chex; + opendir DIR, $chex or die $!; + while (readdir DIR) { + next unless /^ep/; + $ep = $_; + last; + } + closedir DIR; + + my @epoch = split("-", $ep); + $name .= "-" . $epoch[0]; + + if (-e "output/" . $name . ".mp3") { + return 1; + } + + print $name . "\n"; + if ($opt_l) { + return 0; + } + system('/bin/bash', 'mix.sh', $path, $name); + return 0; +} + +opendir RESULTS, ("results/") or die $!; +my @results = sort {(stat $a)[10] <=> (stat $b)[10]} readdir(RESULTS); +closedir RESULTS; + +for my $result (@results) { + next if $result !~ /exp:/; + my $rv = process($result); + #last if $rv; +} diff --git a/mix.sh b/mix.sh index c212cfa..821d802 100755 --- a/mix.sh +++ b/mix.sh @@ -9,6 +9,8 @@ dir=$1 name=$2 now=`date +'%Y%m%d'` +echo "rendering $name" + cd "$dir/samples" mkdir "$name" ../../xfade.sh -- cgit v1.2.3-70-g09d2 From 05401dc3db9414986a2a1477e1e5fa08778da45c Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Tue, 10 Apr 2018 10:45:47 +0200 Subject: latest renders with -l, get.pl to wget from asdf.us --- datasets/get.pl | 4 ++++ latest.pl | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100755 datasets/get.pl diff --git a/datasets/get.pl b/datasets/get.pl new file mode 100755 index 0000000..8b821c6 --- /dev/null +++ b/datasets/get.pl @@ -0,0 +1,4 @@ +#!/usr/bin/perl + +foreach (@ARGV){ system('/usr/bin/wget', 'https://asdf.us/neural/' . $_); } + diff --git a/latest.pl b/latest.pl index b20ca1c..1e3a46e 100644 --- a/latest.pl +++ b/latest.pl @@ -26,14 +26,20 @@ sub process($){ my @epoch = split("-", $ep); $name .= "-" . $epoch[0]; + if (! $epoch[0]) { + return 0; + } if (-e "output/" . $name . ".mp3") { return 1; } print $name . "\n"; - if ($opt_l) { + if (!$opt_l) { return 0; } + print "\n"; + print "_______________________________________________________________\n"; + print "\n"; system('/bin/bash', 'mix.sh', $path, $name); return 0; } -- cgit v1.2.3-70-g09d2 From efe9c4a7f53414c628a865207c6a47f8e5a016f0 Mon Sep 17 00:00:00 2001 From: jules Date: Tue, 10 Apr 2018 10:46:12 +0200 Subject: gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 8ab7957..ca60642 100644 --- a/.gitignore +++ b/.gitignore @@ -107,3 +107,5 @@ run_here* node_modules/ +results/old_checkpoints/ + -- cgit v1.2.3-70-g09d2 From 0ecd9fcea8e7426c003baadc0cd67b62e5cf382f Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Tue, 10 Apr 2018 12:40:06 +0200 Subject: ffmpeg less verbose, get should append suffix --- datasets/get.pl | 3 ++- datasets/split44k.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/datasets/get.pl b/datasets/get.pl index 8b821c6..531b9f7 100755 --- a/datasets/get.pl +++ b/datasets/get.pl @@ -1,4 +1,5 @@ #!/usr/bin/perl -foreach (@ARGV){ system('/usr/bin/wget', 'https://asdf.us/neural/' . $_); } +foreach (@ARGV){ + my $s = $_; if ($s !~/\..*$/) { $s .= ".wav"; } system('/usr/bin/wget', 'https://neural:spawn5@asdf.us/neural/' . $s); } diff --git a/datasets/split44k.sh b/datasets/split44k.sh index 4884af1..b679d29 100755 --- a/datasets/split44k.sh +++ b/datasets/split44k.sh @@ -11,7 +11,7 @@ dataset_path=$3 converted=".temp2.wav" rm -f $converted -ffmpeg -i $fn -ac 1 -ar 44100 $converted +ffmpeg -hide_banner -loglevel error -i $fn -ac 1 -ar 44100 $converted mkdir $dataset_path length=$(ffprobe -i $converted -show_entries format=duration -v quiet -of csv="p=0") -- cgit v1.2.3-70-g09d2 From 6690a335a13a2cb07e78a5c48b8b838de44e3283 Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Tue, 10 Apr 2018 14:42:11 +0200 Subject: perms --- epoch.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 epoch.sh diff --git a/epoch.sh b/epoch.sh old mode 100644 new mode 100755 -- cgit v1.2.3-70-g09d2 From f3d8faecdcf0de97e8027ca9d80315b58fab700a Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Tue, 10 Apr 2018 15:10:24 +0200 Subject: ignore results that are insufficiently trained --- latest.pl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/latest.pl b/latest.pl index 1e3a46e..f1d1aa9 100644 --- a/latest.pl +++ b/latest.pl @@ -29,6 +29,10 @@ sub process($){ if (! $epoch[0]) { return 0; } + $epoch[0] =~ /(\d+)/; + if ($1 < 4) { + return 0; + } if (-e "output/" . $name . ".mp3") { return 1; } -- cgit v1.2.3-70-g09d2 From 3ea2f8f581a857920b7c2f4d1d3a0a12401758ec Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Tue, 10 Apr 2018 15:20:00 +0200 Subject: build --- latest.pl | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/latest.pl b/latest.pl index f1d1aa9..33bfb73 100644 --- a/latest.pl +++ b/latest.pl @@ -23,6 +23,20 @@ sub process($){ } closedir DIR; + my $sampz = $path . "samples/"; + return 0 unless -e $sampz; + my $c = 0; + opendir DIR, $sampz or die $!; + while (readdir DIR) { + next unless /wav$/; + $c += 1; + last; + } + closedir DIR; + if ($c == 0) { + return 0; + } + my @epoch = split("-", $ep); $name .= "-" . $epoch[0]; @@ -37,13 +51,14 @@ sub process($){ return 1; } - print $name . "\n"; if (!$opt_l) { return 0; } print "\n"; print "_______________________________________________________________\n"; print "\n"; + print $name . ", $c samples\n"; + print "\n"; system('/bin/bash', 'mix.sh', $path, $name); return 0; } -- cgit v1.2.3-70-g09d2 From 332d6f5b739c8e037e3d2877085f45fbeef375e8 Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Tue, 10 Apr 2018 15:57:54 +0200 Subject: get script should build dataset --- datasets/get.pl | 6 +++++- latest.pl | 0 2 files changed, 5 insertions(+), 1 deletion(-) mode change 100644 => 100755 latest.pl diff --git a/datasets/get.pl b/datasets/get.pl index 531b9f7..14c5a5b 100755 --- a/datasets/get.pl +++ b/datasets/get.pl @@ -1,5 +1,9 @@ #!/usr/bin/perl foreach (@ARGV){ - my $s = $_; if ($s !~/\..*$/) { $s .= ".wav"; } system('/usr/bin/wget', 'https://neural:spawn5@asdf.us/neural/' . $s); } + my $s = $_; + if ($s !~/\..*$/) { $s .= ".wav"; } + system('/usr/bin/wget', 'https://neural:spawn5@asdf.us/neural/' . $s); + system('/usr/bin/perl', 'dataset.pl', $s); +} diff --git a/latest.pl b/latest.pl old mode 100644 new mode 100755 -- cgit v1.2.3-70-g09d2 From b8ce4da58bef73ce4adb7c278803fcef37b9c2ad Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Tue, 10 Apr 2018 16:29:11 +0200 Subject: latest --- latest.pl | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/latest.pl b/latest.pl index 33bfb73..cd86de2 100755 --- a/latest.pl +++ b/latest.pl @@ -3,8 +3,9 @@ use strict; use Getopt::Std; -our $opt_l; -getopts('l'); +our $opt_l; # mix and upload files +our $opt_v; # print log for all files +getopts('lav'); # my $s = "results/exp:03_responses-frame_sizes:8,2-n_rnn:2-dataset:03_responses/checkpoints/best-ep1-it1035"; sub process($){ @@ -14,6 +15,7 @@ sub process($){ my $ep; my $path = "results/" . $dir . "/"; my $chex = $path . "checkpoints/"; + my $log = $path . 'log'; return 0 unless -e $chex; opendir DIR, $chex or die $!; while (readdir DIR) { @@ -47,6 +49,11 @@ sub process($){ if ($1 < 4) { return 0; } + + if ($opt_v) { + print `grep valid $log`; + } + if (-e "output/" . $name . ".mp3") { return 1; } -- cgit v1.2.3-70-g09d2 From a1c16f063f7d753909c75ca9cbac459af7eb8245 Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Tue, 10 Apr 2018 16:29:50 +0200 Subject: latest --- latest.pl | 1 + 1 file changed, 1 insertion(+) diff --git a/latest.pl b/latest.pl index cd86de2..2218d06 100755 --- a/latest.pl +++ b/latest.pl @@ -51,6 +51,7 @@ sub process($){ } if ($opt_v) { + print $name; print `grep valid $log`; } -- cgit v1.2.3-70-g09d2 From 48f321017c86097d7500318f748a90959f5fafba Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Tue, 10 Apr 2018 16:30:45 +0200 Subject: latest --- latest.pl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/latest.pl b/latest.pl index 2218d06..a29cf50 100755 --- a/latest.pl +++ b/latest.pl @@ -35,6 +35,12 @@ sub process($){ last; } closedir DIR; + + if ($opt_v) { + print $name . "\n"; + print `grep valid $log`; + } + if ($c == 0) { return 0; } @@ -50,11 +56,6 @@ sub process($){ return 0; } - if ($opt_v) { - print $name; - print `grep valid $log`; - } - if (-e "output/" . $name . ".mp3") { return 1; } -- cgit v1.2.3-70-g09d2 From fb5da04e2ec639c8791a400dd9a748de1484d89f Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Tue, 10 Apr 2018 16:32:20 +0200 Subject: stat 9 --- latest.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/latest.pl b/latest.pl index a29cf50..f1a36a2 100755 --- a/latest.pl +++ b/latest.pl @@ -73,7 +73,7 @@ sub process($){ } opendir RESULTS, ("results/") or die $!; -my @results = sort {(stat $a)[10] <=> (stat $b)[10]} readdir(RESULTS); +my @results = sort {(stat $a)[9] <=> (stat $b)[9]} readdir(RESULTS); closedir RESULTS; for my $result (@results) { -- cgit v1.2.3-70-g09d2 From 7c0c0850ed7de27e42594b1862ba18e55cbe4221 Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Tue, 10 Apr 2018 16:34:01 +0200 Subject: latest --- latest.pl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/latest.pl b/latest.pl index f1a36a2..6fd6dcd 100755 --- a/latest.pl +++ b/latest.pl @@ -73,7 +73,8 @@ sub process($){ } opendir RESULTS, ("results/") or die $!; -my @results = sort {(stat $a)[9] <=> (stat $b)[9]} readdir(RESULTS); +# my @results = sort {(stat $a)[9] <=> (stat $b)[9]} readdir(RESULTS); +my @results = sort {$a cmp $b} readdir(RESULTS); closedir RESULTS; for my $result (@results) { -- cgit v1.2.3-70-g09d2 From 860acac9862cd5ebe80d2a428d446dbdcc41d197 Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Tue, 10 Apr 2018 16:34:41 +0200 Subject: latest --- latest.pl | 1 + 1 file changed, 1 insertion(+) diff --git a/latest.pl b/latest.pl index 6fd6dcd..60d4a06 100755 --- a/latest.pl +++ b/latest.pl @@ -39,6 +39,7 @@ sub process($){ if ($opt_v) { print $name . "\n"; print `grep valid $log`; + print "\n"; } if ($c == 0) { -- cgit v1.2.3-70-g09d2 From 805a4fc2b3e406e0b1506fdfd54aa8750d289659 Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Tue, 10 Apr 2018 17:49:13 +0200 Subject: counting samples again --- latest.pl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/latest.pl b/latest.pl index 60d4a06..4975a0a 100755 --- a/latest.pl +++ b/latest.pl @@ -32,7 +32,6 @@ sub process($){ while (readdir DIR) { next unless /wav$/; $c += 1; - last; } closedir DIR; @@ -52,6 +51,11 @@ sub process($){ if (! $epoch[0]) { return 0; } + + if (! $opt_l) { + print $name . ", $c samples\n"; + } + $epoch[0] =~ /(\d+)/; if ($1 < 4) { return 0; -- cgit v1.2.3-70-g09d2 From f8230525f125df5d80e2d58091ec5231c0e16cdc Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Tue, 10 Apr 2018 19:33:45 +0200 Subject: print filenames --- upload.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/upload.js b/upload.js index 1cdcba8..6bbb71e 100644 --- a/upload.js +++ b/upload.js @@ -24,10 +24,15 @@ if (!argv['_'] || ! argv['_'].length) { var r = request.post('https://bucky.asdf.us/api/thread/' + id + '/comment', function (err, res, body) { if (err) { - return console.error('upload failed:', err); + return console.error('Upload failed:', err); } // console.log(err, res, body) - console.log('Upload successful'); + console.log('\n') + console.log('Upload successful!') + console.log('\n') + argv['_'].forEach(fn => { + console.log('https://s3.amazonaws.com/i.asdf.us/bucky/data/' + id + '/' + fn); + }) }) var form = r.form() -- cgit v1.2.3-70-g09d2 From d559edfb51c7a28f9b3d426f652f90215fef5e26 Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Tue, 10 Apr 2018 21:12:28 +0200 Subject: upload log dogg --- results/xfade.sh | 2 +- upload.js | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/results/xfade.sh b/results/xfade.sh index 9fc4d39..0a5e34a 100755 --- a/results/xfade.sh +++ b/results/xfade.sh @@ -15,7 +15,7 @@ do cp $file mix.wav else # ../../crossfade_cat.sh $crossfade_dur mix.wav $file yes yes - echo $file + # echo $file sox $file tmp.wav highpass 10 sox mix.wav tmp.wav out.wav splice $(soxi -D mix.wav),0.01 mv out.wav mix.wav diff --git a/upload.js b/upload.js index 6bbb71e..ca15029 100644 --- a/upload.js +++ b/upload.js @@ -31,7 +31,8 @@ var r = request.post('https://bucky.asdf.us/api/thread/' + id + '/comment', func console.log('Upload successful!') console.log('\n') argv['_'].forEach(fn => { - console.log('https://s3.amazonaws.com/i.asdf.us/bucky/data/' + id + '/' + fn); + const partz = fn.split('/') + console.log('https://s3.amazonaws.com/i.asdf.us/bucky/data/' + id + '/' + partz[partz.length-1]); }) }) -- cgit v1.2.3-70-g09d2 From 33bcbba9cf576bcae3fe72732a373d66af177591 Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Tue, 10 Apr 2018 21:48:43 +0200 Subject: dont put flacs in git --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ca60642..2ad71be 100644 --- a/.gitignore +++ b/.gitignore @@ -97,6 +97,7 @@ ENV/ *.mp3 *.aif *.aiff +*.flac results/exp* !results/*.sh -- cgit v1.2.3-70-g09d2 From a04b7b8117ee6993bba35ea99fbd9f2530213e2d Mon Sep 17 00:00:00 2001 From: jules Date: Wed, 11 Apr 2018 11:26:32 +0200 Subject: specify which experiment to process --- latest.pl | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/latest.pl b/latest.pl index 4975a0a..c09a5cd 100755 --- a/latest.pl +++ b/latest.pl @@ -7,6 +7,14 @@ our $opt_l; # mix and upload files our $opt_v; # print log for all files getopts('lav'); +our $matches = 0; +if (scalar @ARGV) { + $matches = {}; + for (@ARGV) { + my ($name, $ep) = split("-", $_, 2); + $matches->{$name} = 1; + } +} # my $s = "results/exp:03_responses-frame_sizes:8,2-n_rnn:2-dataset:03_responses/checkpoints/best-ep1-it1035"; sub process($){ my ($dir) = @_; @@ -35,6 +43,10 @@ sub process($){ } closedir DIR; + if ($matches && ! exists $matches->{$name}) { + return 0; + } + if ($opt_v) { print $name . "\n"; print `grep valid $log`; -- cgit v1.2.3-70-g09d2 From d9f28aa4b5c538468f8b3b8cf39189afc3d13ca5 Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Thu, 12 Apr 2018 15:53:25 +0200 Subject: highpass on first wav --- datasets/dataset.pl | 4 ++-- mix.sh | 5 ++--- results/xfade.sh | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/datasets/dataset.pl b/datasets/dataset.pl index 8b77af9..edde43f 100755 --- a/datasets/dataset.pl +++ b/datasets/dataset.pl @@ -48,7 +48,7 @@ sub process($) { my $a_tmp = "a_" . $filename; my $b_tmp = "b_" . $filename; - if ($opt_c) { + if (!$opt_c) { print "Normalizing..."; system("sox", "-v", 0.945, $filename, $a_tmp); } else { @@ -56,7 +56,7 @@ sub process($) { } system("./spread.sh", $a_tmp, $b_tmp, 0.999, $scale, 1.001); system("./split44k.sh", $b_tmp, 8, $name); - if ($opt_c) { + if (!$opt_c) { system("/bin/rm", $a_tmp); } system("/bin/rm", $b_tmp); diff --git a/mix.sh b/mix.sh index 821d802..9df81a7 100755 --- a/mix.sh +++ b/mix.sh @@ -14,9 +14,8 @@ echo "rendering $name" cd "$dir/samples" mkdir "$name" ../../xfade.sh -sox mix.wav norm.wav norm -lame -V 0 norm.wav "../../../output/$name.mp3" -rm mix.wav norm.wav +lame -V 0 mix.wav "../../../output/$name.mp3" +rm mix.wav mv *.wav "$name" cd .. grep valid log diff --git a/results/xfade.sh b/results/xfade.sh index 0a5e34a..55d343d 100755 --- a/results/xfade.sh +++ b/results/xfade.sh @@ -12,7 +12,7 @@ do if [ $i -eq 1 ] then - cp $file mix.wav + sox $file mix.wav highpass 10 else # ../../crossfade_cat.sh $crossfade_dur mix.wav $file yes yes # echo $file -- cgit v1.2.3-70-g09d2 From 48d9f2e2398014d39f818542795a4e176d5207a6 Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Fri, 13 Apr 2018 12:03:49 +0200 Subject: get script in root dir --- get.pl | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100755 get.pl diff --git a/get.pl b/get.pl new file mode 100755 index 0000000..4447ab3 --- /dev/null +++ b/get.pl @@ -0,0 +1,17 @@ +#!/usr/bin/perl + +$SIG{TERM} = $SIG{INT} = sub { exit 1 }; + +chdir('datasets'); + +foreach (@ARGV){ + my $s = $_; + if ($s !~/\..*$/) { $s .= ".wav"; } + system('/usr/bin/wget', 'https://neural:spawn5@asdf.us/neural/' . $s); + system('/usr/bin/perl', 'dataset.pl', $s); +} + +END { + chdir('..'); +} + -- cgit v1.2.3-70-g09d2 From 55cdfa754eea6356ef425afdd2c374886be992ad Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Fri, 13 Apr 2018 17:23:32 +0200 Subject: render mp3 automatically and send the link to slack --- upload.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/upload.js b/upload.js index ca15029..f969109 100644 --- a/upload.js +++ b/upload.js @@ -1,5 +1,7 @@ #!`which node` +const { execFile } = require('child_process'); + require('dotenv').load(); var request = require('request') var path = require('path') @@ -32,7 +34,14 @@ var r = request.post('https://bucky.asdf.us/api/thread/' + id + '/comment', func console.log('\n') argv['_'].forEach(fn => { const partz = fn.split('/') - console.log('https://s3.amazonaws.com/i.asdf.us/bucky/data/' + id + '/' + partz[partz.length-1]); + const url = 'https://s3.amazonaws.com/i.asdf.us/bucky/data/' + id + '/' + partz[partz.length-1] + console.log(url) + const child = execFile('slack', ['chat', 'send', url, '#ai'], (error, stdout, stderr) => { + if (error) { + throw error; + } + console.log(stdout); + }); }) }) -- cgit v1.2.3-70-g09d2 From 756e4d11226dfd59a3da49e571f605d29e715170 Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Thu, 19 Apr 2018 16:52:21 +0200 Subject: wav export scripts --- export-all.pl | 28 ++++++++++++++++++++++++++++ mix-wav.sh | 19 +++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 export-all.pl create mode 100644 mix-wav.sh diff --git a/export-all.pl b/export-all.pl new file mode 100644 index 0000000..48670b5 --- /dev/null +++ b/export-all.pl @@ -0,0 +1,28 @@ +#!/usr/bin/perl + +use strict; + +sub process($){ + my ($dir) = @_; + my @exp = split(":", $dir); + my $name = $exp[4]; + my $ep; + my $path = "results/" . $dir . "/"; + my $chex = $path . "checkpoints/"; + return 0 unless -e $chex; + my $sampz = $path . "samples/"; + return 0 unless -e $sampz; + + print $name . "\n"; + system('/bin/bash', 'mix.sh', $path, $name); +} + +opendir RESULTS, ("results/") or die $!; +# my @results = sort {(stat $a)[9] <=> (stat $b)[9]} readdir(RESULTS); +my @results = sort {$a cmp $b} readdir(RESULTS); +closedir RESULTS; + +for my $result (@results) { + next if $result !~ /exp:/; + process($result); +} diff --git a/mix-wav.sh b/mix-wav.sh new file mode 100644 index 0000000..663d50a --- /dev/null +++ b/mix-wav.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 render_name" + exit +fi + +dir=$1 +name=$2 +now=`date +'%Y%m%d'` + +echo "rendering $name" + +cd "$dir/samples" +mv */*.wav . +../../xfade.sh +sox mix.wav norm.wav norm +mv norm.wav "../../../wav/${name}.wav" +cd ../../.. -- cgit v1.2.3-70-g09d2 From 70502d8b2dc626d0a94e6bcde47383d3f4419404 Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Thu, 19 Apr 2018 16:58:55 +0200 Subject: diff export --- mix-wav.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/mix-wav.sh b/mix-wav.sh index 663d50a..ceb4394 100644 --- a/mix-wav.sh +++ b/mix-wav.sh @@ -12,8 +12,14 @@ now=`date +'%Y%m%d'` echo "rendering $name" cd "$dir/samples" -mv */*.wav . -../../xfade.sh -sox mix.wav norm.wav norm -mv norm.wav "../../../wav/${name}.wav" +for i in * +do + if [[ -d $$i ]]; then + cd $i + echo $i + #../../../xfade.sh + #sox mix.wav norm.wav norm + #mv norm.wav "../../../../wav/${i}.wav" + cd .. + fi cd ../../.. -- cgit v1.2.3-70-g09d2 From 0729aaaf19126e7189bfe05aa84c7143568ad30a Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Thu, 19 Apr 2018 17:01:28 +0200 Subject: mix path --- export-all.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/export-all.pl b/export-all.pl index 48670b5..a02ecc5 100644 --- a/export-all.pl +++ b/export-all.pl @@ -14,7 +14,7 @@ sub process($){ return 0 unless -e $sampz; print $name . "\n"; - system('/bin/bash', 'mix.sh', $path, $name); + system('/bin/bash', 'mix-wav.sh', $path, $name); } opendir RESULTS, ("results/") or die $!; -- cgit v1.2.3-70-g09d2 From 49d5deae29bafd4a98e36521cf7921b504160195 Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Thu, 19 Apr 2018 17:01:59 +0200 Subject: mix path --- export-all.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/export-all.pl b/export-all.pl index a02ecc5..e840729 100644 --- a/export-all.pl +++ b/export-all.pl @@ -13,7 +13,7 @@ sub process($){ my $sampz = $path . "samples/"; return 0 unless -e $sampz; - print $name . "\n"; + # print $name . "\n"; system('/bin/bash', 'mix-wav.sh', $path, $name); } -- cgit v1.2.3-70-g09d2 From a14e2d39eb71001963702660628473cbf4864fac Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Thu, 19 Apr 2018 17:02:26 +0200 Subject: mix path --- mix-wav.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/mix-wav.sh b/mix-wav.sh index ceb4394..46a1a66 100644 --- a/mix-wav.sh +++ b/mix-wav.sh @@ -22,4 +22,5 @@ do #mv norm.wav "../../../../wav/${i}.wav" cd .. fi +done cd ../../.. -- cgit v1.2.3-70-g09d2 From 211890f17e5db060d1850abf1f573b52e3903d78 Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Thu, 19 Apr 2018 17:03:02 +0200 Subject: mix path --- mix-wav.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mix-wav.sh b/mix-wav.sh index 46a1a66..40e63a8 100644 --- a/mix-wav.sh +++ b/mix-wav.sh @@ -17,9 +17,9 @@ do if [[ -d $$i ]]; then cd $i echo $i - #../../../xfade.sh - #sox mix.wav norm.wav norm - #mv norm.wav "../../../../wav/${i}.wav" + ../../../xfade.sh + sox mix.wav norm.wav norm + mv norm.wav "../../../../wav/${i}.wav" cd .. fi done -- cgit v1.2.3-70-g09d2 From 84e98d5cc446e5b3a75dd4f8504a9acca7b8ad10 Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Thu, 19 Apr 2018 17:04:18 +0200 Subject: mix path --- mix-wav.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix-wav.sh b/mix-wav.sh index 40e63a8..23c81ec 100644 --- a/mix-wav.sh +++ b/mix-wav.sh @@ -14,7 +14,7 @@ echo "rendering $name" cd "$dir/samples" for i in * do - if [[ -d $$i ]]; then + if [[ -d $i ]]; then cd $i echo $i ../../../xfade.sh -- cgit v1.2.3-70-g09d2 From 2fc8e30aafce16bfd103a8ea59aeb6258b433c2f Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Thu, 19 Apr 2018 17:04:24 +0200 Subject: mix path --- mix-wav.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mix-wav.sh b/mix-wav.sh index 23c81ec..2527f33 100644 --- a/mix-wav.sh +++ b/mix-wav.sh @@ -17,9 +17,9 @@ do if [[ -d $i ]]; then cd $i echo $i - ../../../xfade.sh - sox mix.wav norm.wav norm - mv norm.wav "../../../../wav/${i}.wav" + # ../../../xfade.sh + # sox mix.wav norm.wav norm + # mv norm.wav "../../../../wav/${i}.wav" cd .. fi done -- cgit v1.2.3-70-g09d2 From 3a4c87d43e96884b71be2b4292ebf6d296794fe5 Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Thu, 19 Apr 2018 17:04:57 +0200 Subject: mix path --- mix-wav.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix-wav.sh b/mix-wav.sh index 2527f33..f6fc821 100644 --- a/mix-wav.sh +++ b/mix-wav.sh @@ -9,7 +9,7 @@ dir=$1 name=$2 now=`date +'%Y%m%d'` -echo "rendering $name" +# echo "rendering $name" cd "$dir/samples" for i in * -- cgit v1.2.3-70-g09d2 From 54fa412397145987ec17c38d5772535400c1e937 Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Thu, 19 Apr 2018 17:05:06 +0200 Subject: mix path --- mix-wav.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mix-wav.sh b/mix-wav.sh index f6fc821..5348309 100644 --- a/mix-wav.sh +++ b/mix-wav.sh @@ -17,9 +17,9 @@ do if [[ -d $i ]]; then cd $i echo $i - # ../../../xfade.sh - # sox mix.wav norm.wav norm - # mv norm.wav "../../../../wav/${i}.wav" + ../../../xfade.sh + sox mix.wav norm.wav norm + mv norm.wav "../../../../wav/${i}.wav" cd .. fi done -- cgit v1.2.3-70-g09d2 From 7287a81b9af14ca8b24f258a2e3de08f39bb63c7 Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Tue, 1 May 2018 03:22:09 +0200 Subject: process script --- .gitignore | 3 ++ datasets/dataset.pl | 4 ++ datasets/silence.sh | 0 latest.pl | 6 +-- process_fns | 109 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 119 insertions(+), 3 deletions(-) create mode 100644 datasets/silence.sh create mode 100755 process_fns diff --git a/.gitignore b/.gitignore index 2ad71be..26a1d76 100644 --- a/.gitignore +++ b/.gitignore @@ -110,3 +110,6 @@ node_modules/ results/old_checkpoints/ +mgb +run_* + diff --git a/datasets/dataset.pl b/datasets/dataset.pl index edde43f..a6eb984 100755 --- a/datasets/dataset.pl +++ b/datasets/dataset.pl @@ -25,6 +25,10 @@ sub process($) { return; } + if ($ext eq 'mp3') { + system('ffmpeg', '-i', $filename, $name . '.wav') + $filename = $name . '.wav'; + } my $soxi = `soxi $filename`; my @lines = split("\n", $soxi); diff --git a/datasets/silence.sh b/datasets/silence.sh new file mode 100644 index 0000000..e69de29 diff --git a/latest.pl b/latest.pl index c09a5cd..ddd5952 100755 --- a/latest.pl +++ b/latest.pl @@ -69,9 +69,9 @@ sub process($){ } $epoch[0] =~ /(\d+)/; - if ($1 < 4) { - return 0; - } + #if ($1 < 4) { + # return 0; + #} if (-e "output/" . $name . ".mp3") { return 1; diff --git a/process_fns b/process_fns new file mode 100755 index 0000000..ad47a28 --- /dev/null +++ b/process_fns @@ -0,0 +1,109 @@ +workon samplernn + +function generate () { + exp_name=$1 + n_samples=$2 + sample_rate=$3 + duration=$4 + let sample_length=$3*$4 + + echo "" + echo "###################################################" + echo "###################################################" + echo "###################################################" + echo "" + echo ">> generating $exp_name" + echo "" + echo "###################################################" + echo "###################################################" + echo "###################################################" + echo "" + python generate.py \ + --exp $exp_name --dataset $exp_name \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --n_samples $n_samples \ + --epoch_limit 1 \ + --resume True + sleep 1 +} +function run () { + exp_name=$1 + epoch_limit=$2 + n_samples=$3 + sample_rate=$4 + duration=$5 + let sample_length=$4*$5 + + echo "" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo "" + echo ">> running $exp_name" + echo "" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + echo "" + python train.py \ + --exp $exp_name --dataset $exp_name \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --n_samples $n_samples \ + --keep_old_checkpoints False \ + --epoch_limit $epoch_limit \ + --resume True + sleep 1 +} +function standard () { + dataset=$1 + run $1 6 6 44100 5 + ./latest.pl -l $1 +} +function quick () { + dataset=$1 + run $1 4 6 44100 5 + ./latest.pl -l $1 +} +function half () { + dataset=$1 + run $1 2 6 44100 5 + ./latest.pl -l $1 +} +function fast () { + dataset=$1 + run $1 1 6 44100 10 + ./latest.pl -l $1 +} +function emph () { + dataset=$1 + run $1 1 6 44100 10 + run $1 3 6 44100 5 + ./latest.pl -l $1 +} + +function start () { + dataset=$1 + cd datasets/ + perl dataset.pl "${dataset}.wav" + cd .. + quick $dataset +} +function start_standard () { + dataset=$1 + cd datasets/ + perl dataset.pl "${dataset}.wav" + cd .. + standard $dataset +} + -- cgit v1.2.3-70-g09d2 From 02c27ad8dcc689eda3b3e68d260cb23e9bf9a3cf Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Tue, 1 May 2018 10:47:24 +0200 Subject: fix upload --- datasets/dataset.pl | 2 +- upload.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) mode change 100644 => 100755 upload.js diff --git a/datasets/dataset.pl b/datasets/dataset.pl index a6eb984..75aa2b1 100755 --- a/datasets/dataset.pl +++ b/datasets/dataset.pl @@ -26,7 +26,7 @@ sub process($) { } if ($ext eq 'mp3') { - system('ffmpeg', '-i', $filename, $name . '.wav') + system('ffmpeg', '-i', $filename, $name . '.wav'); $filename = $name . '.wav'; } my $soxi = `soxi $filename`; diff --git a/upload.js b/upload.js old mode 100644 new mode 100755 index f969109..73e1147 --- a/upload.js +++ b/upload.js @@ -1,4 +1,4 @@ -#!`which node` +#!/home/spawn/.nvm/versions/node/v9.11.1/bin/node const { execFile } = require('child_process'); -- cgit v1.2.3-70-g09d2 From 98affbe62185cf425a629ae68528f5381f3a864a Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Tue, 1 May 2018 12:11:46 +0200 Subject: boilerplate for a slack bot --- package-lock.json | 604 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + spawn.js | 173 ++++++++++++++++ 3 files changed, 778 insertions(+) create mode 100644 spawn.js diff --git a/package-lock.json b/package-lock.json index cefb77b..f53f640 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,6 +4,127 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "@sindresorhus/is": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.7.0.tgz", + "integrity": "sha512-ONhaKPIufzzrlNbqtWFFd+jlnemX6lJAgq9ZeiZtS7I1PIf/la7CW4m83rTXRnVnsMbW2k56pGYu7AUFJD9Pow==" + }, + "@slack/client": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@slack/client/-/client-4.2.0.tgz", + "integrity": "sha512-FASaHhy8PPEN/h/eMy9edzwhaRRGfqZA8LklkBcz8z4VYrbWSYljoeup8CfFrl69Atbv4ILrRAPGefytQzoSUA==", + "requires": { + "@types/delay": "2.0.1", + "@types/form-data": "2.2.1", + "@types/got": "7.1.8", + "@types/is-stream": "1.1.0", + "@types/loglevel": "1.5.3", + "@types/node": "9.6.8", + "@types/p-cancelable": "0.3.0", + "@types/p-queue": "2.3.1", + "@types/p-retry": "1.0.1", + "@types/retry": "0.10.2", + "@types/url-join": "0.8.2", + "@types/ws": "4.0.2", + "delay": "2.0.0", + "eventemitter3": "3.1.0", + "finity": "0.5.4", + "form-data": "2.3.2", + "got": "8.3.1", + "is-stream": "1.1.0", + "loglevel": "1.6.1", + "object.entries": "1.0.4", + "object.getownpropertydescriptors": "2.0.3", + "object.values": "1.0.4", + "p-cancelable": "0.3.0", + "p-queue": "2.4.2", + "p-retry": "1.0.0", + "retry": "0.10.1", + "url-join": "4.0.0", + "ws": "4.1.0" + } + }, + "@types/delay": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@types/delay/-/delay-2.0.1.tgz", + "integrity": "sha512-D1/YuYOcdOIdaQnaiUJ77VcilVvESkynw79CtGqpjkXyv4OUezEVZtdXnSOwXL8Zcelu66QbyC8QQcVQ/ZPdig==" + }, + "@types/events": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@types/events/-/events-1.2.0.tgz", + "integrity": "sha512-KEIlhXnIutzKwRbQkGWb/I4HFqBuUykAdHgDED6xqwXJfONCjF5VoE0cXEiurh3XauygxzeDzgtXUqvLkxFzzA==" + }, + "@types/form-data": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/@types/form-data/-/form-data-2.2.1.tgz", + "integrity": "sha512-JAMFhOaHIciYVh8fb5/83nmuO/AHwmto+Hq7a9y8FzLDcC1KCU344XDOMEmahnrTFlHjgh4L0WJFczNIX2GxnQ==", + "requires": { + "@types/node": "9.6.8" + } + }, + "@types/got": { + "version": "7.1.8", + "resolved": "https://registry.npmjs.org/@types/got/-/got-7.1.8.tgz", + "integrity": "sha512-QxcSkx9PjHH7uqbzqKTKOAjGbayyo+dECnnqM3BBMC0WjYAqx0e6Qi9AFR4jluvx91e7qzgz4aGka7AhCTHYRw==", + "requires": { + "@types/node": "9.6.8" + } + }, + "@types/is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@types/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha512-jkZatu4QVbR60mpIzjINmtS1ZF4a/FqdTUTBeQDVOQ2PYyidtwFKr0B5G6ERukKwliq+7mIXvxyppwzG5EgRYg==", + "requires": { + "@types/node": "9.6.8" + } + }, + "@types/loglevel": { + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/@types/loglevel/-/loglevel-1.5.3.tgz", + "integrity": "sha512-TzzIZihV+y9kxSg5xJMkyIkaoGkXi50isZTtGHObNHRqAAwjGNjSCNPI7AUAv0tZUKTq9f2cdkCUd/2JVZUTrA==" + }, + "@types/node": { + "version": "9.6.8", + "resolved": "https://registry.npmjs.org/@types/node/-/node-9.6.8.tgz", + "integrity": "sha512-0PmgMBskTJa7zDyENW9C7Lunk+I0L2CHYF2RrBRljCmLSMM1fBHIIdvE1IboNNz7N6t+raJIj90YMvUYl2VT1g==" + }, + "@types/p-cancelable": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@types/p-cancelable/-/p-cancelable-0.3.0.tgz", + "integrity": "sha512-sP+9Ivnpil7cdmvr5O+145aXm65YX8Y+Lrul1ojdYz6yaE05Dqonn6Z9v5eqJCQ0UeSGcTRtepMlZDh9ywdKgw==" + }, + "@types/p-queue": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@types/p-queue/-/p-queue-2.3.1.tgz", + "integrity": "sha512-JyO7uMAtkcMMULmsTQ4t/lCC8nxirTtweGG1xAFNNIAoC1RemmeIxq8PiKghuEy99XdbS6Lwx4zpbXUjfeSSAA==" + }, + "@types/p-retry": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@types/p-retry/-/p-retry-1.0.1.tgz", + "integrity": "sha512-HgQPG9kkUb4EpTeUv2taH2nBZsVUb5aOTSw3X2YozcTG1ttmGcLaLKx1MbAz1evVfUEDTCAPmdz2HiFztIyWrw==", + "requires": { + "@types/retry": "0.10.2" + } + }, + "@types/retry": { + "version": "0.10.2", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.10.2.tgz", + "integrity": "sha512-LqJkY4VQ7S09XhI7kA3ON71AxauROhSv74639VsNXC9ish4IWHnIi98if+nP1MxQV3RMPqXSCYgpPsDHjlg9UQ==" + }, + "@types/url-join": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/@types/url-join/-/url-join-0.8.2.tgz", + "integrity": "sha1-EYHsvh2XtwNODqHjXmLobMJrQi0=" + }, + "@types/ws": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-4.0.2.tgz", + "integrity": "sha512-tlDVFHCcJdNqYgjGNDPDCo4tNqhFMymIAdJCcykFbdhYr4X6vD7IlMxY0t3/k6Pfup68YNkMTpRfLKTRuKDmnQ==", + "requires": { + "@types/events": "1.2.0", + "@types/node": "9.6.8" + } + }, "ajv": { "version": "5.5.2", "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz", @@ -25,6 +146,11 @@ "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" }, + "async-limiter": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz", + "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg==" + }, "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -57,11 +183,40 @@ "hoek": "4.2.1" } }, + "cacheable-request": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-2.1.4.tgz", + "integrity": "sha1-DYCIAbY0KtM8kd+dC0TcCbkeXD0=", + "requires": { + "clone-response": "1.0.2", + "get-stream": "3.0.0", + "http-cache-semantics": "3.8.1", + "keyv": "3.0.0", + "lowercase-keys": "1.0.0", + "normalize-url": "2.0.1", + "responselike": "1.0.2" + }, + "dependencies": { + "lowercase-keys": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.0.tgz", + "integrity": "sha1-TjNms55/VFfjXxMkvfb4jQv8cwY=" + } + } + }, "caseless": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" }, + "clone-response": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.2.tgz", + "integrity": "sha1-0dyXOSAxTfZ/vrlCI7TuNQI56Ws=", + "requires": { + "mimic-response": "1.0.0" + } + }, "co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", @@ -106,6 +261,36 @@ "assert-plus": "1.0.0" } }, + "decode-uri-component": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", + "integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=" + }, + "decompress-response": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", + "integrity": "sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=", + "requires": { + "mimic-response": "1.0.0" + } + }, + "define-properties": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.2.tgz", + "integrity": "sha1-g6c/L+pWmJj7c3GTyPhzyvbUXJQ=", + "requires": { + "foreach": "2.0.5", + "object-keys": "1.0.11" + } + }, + "delay": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/delay/-/delay-2.0.0.tgz", + "integrity": "sha1-kRLq3APk7H4AKXM3iW8nO72R+uU=", + "requires": { + "p-defer": "1.0.0" + } + }, "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -116,6 +301,11 @@ "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-5.0.1.tgz", "integrity": "sha512-4As8uPrjfwb7VXC+WnLCbXK7y+Ueb2B3zgNCePYfhxS1PYeaO1YTeplffTEcbfLhvFNGLAz90VvJs9yomG7bow==" }, + "duplexer3": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz", + "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=" + }, "ecc-jsbn": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz", @@ -125,6 +315,33 @@ "jsbn": "0.1.1" } }, + "es-abstract": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.11.0.tgz", + "integrity": "sha512-ZnQrE/lXTTQ39ulXZ+J1DTFazV9qBy61x2bY071B+qGco8Z8q1QddsLdt/EF8Ai9hcWH72dWS0kFqXLxOxqslA==", + "requires": { + "es-to-primitive": "1.1.1", + "function-bind": "1.1.1", + "has": "1.0.1", + "is-callable": "1.1.3", + "is-regex": "1.0.4" + } + }, + "es-to-primitive": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.1.1.tgz", + "integrity": "sha1-RTVSSKiJeQNLZ5Lhm7gfK3l13Q0=", + "requires": { + "is-callable": "1.1.3", + "is-date-object": "1.0.1", + "is-symbol": "1.0.1" + } + }, + "eventemitter3": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.0.tgz", + "integrity": "sha512-ivIvhpq/Y0uSjcHDcOIccjmYjGLcP09MFGE7ysAwkAvkXfpZlC985pH2/ui64DKazbTW/4kN3yqozUxlXzI6cA==" + }, "extend": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.1.tgz", @@ -145,6 +362,16 @@ "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" }, + "finity": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/finity/-/finity-0.5.4.tgz", + "integrity": "sha512-3l+5/1tuw616Lgb0QBimxfdd2TqaDGpfCBpfX6EqtFmqUV3FtQnVEX4Aa62DagYEqnsTIjZcTfbq9msDbXYgyA==" + }, + "foreach": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.5.tgz", + "integrity": "sha1-C+4AUBiusmDQo6865ljdATbsG5k=" + }, "forever-agent": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", @@ -160,6 +387,25 @@ "mime-types": "2.1.18" } }, + "from2": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz", + "integrity": "sha1-i/tVAr3kpNNs/e6gB/zKIdfjgq8=", + "requires": { + "inherits": "2.0.3", + "readable-stream": "2.3.6" + } + }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + }, + "get-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=" + }, "getpass": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", @@ -168,6 +414,37 @@ "assert-plus": "1.0.0" } }, + "got": { + "version": "8.3.1", + "resolved": "https://registry.npmjs.org/got/-/got-8.3.1.tgz", + "integrity": "sha512-tiLX+bnYm5A56T5N/n9Xo89vMaO1mrS9qoDqj3u/anVooqGozvY/HbXzEpDfbNeKsHCBpK40gSbz8wGYSp3i1w==", + "requires": { + "@sindresorhus/is": "0.7.0", + "cacheable-request": "2.1.4", + "decompress-response": "3.3.0", + "duplexer3": "0.1.4", + "get-stream": "3.0.0", + "into-stream": "3.1.0", + "is-retry-allowed": "1.1.0", + "isurl": "1.0.0", + "lowercase-keys": "1.0.1", + "mimic-response": "1.0.0", + "p-cancelable": "0.4.1", + "p-timeout": "2.0.1", + "pify": "3.0.0", + "safe-buffer": "5.1.1", + "timed-out": "4.0.1", + "url-parse-lax": "3.0.0", + "url-to-options": "1.0.1" + }, + "dependencies": { + "p-cancelable": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-0.4.1.tgz", + "integrity": "sha512-HNa1A8LvB1kie7cERyy21VNeHb2CWJJYqyyC2o3klWFfMGlFmWv2Z7sFgZH8ZiaYL95ydToKTFVXgMV/Os0bBQ==" + } + } + }, "har-schema": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", @@ -182,6 +459,27 @@ "har-schema": "2.0.0" } }, + "has": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.1.tgz", + "integrity": "sha1-hGFzP1OLCDfJNh45qauelwTcLyg=", + "requires": { + "function-bind": "1.1.1" + } + }, + "has-symbol-support-x": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/has-symbol-support-x/-/has-symbol-support-x-1.4.2.tgz", + "integrity": "sha512-3ToOva++HaW+eCpgqZrCfN51IPB+7bJNVT6CUATzueB5Heb8o6Nam0V3HG5dlDvZU1Gn5QLcbahiKw/XVk5JJw==" + }, + "has-to-string-tag-x": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/has-to-string-tag-x/-/has-to-string-tag-x-1.4.1.tgz", + "integrity": "sha512-vdbKfmw+3LoOYVr+mtxHaX5a96+0f3DljYd8JOqvOLsf5mw2Otda2qCDT9qRqLAhrjyQ0h7ual5nOiASpsGNFw==", + "requires": { + "has-symbol-support-x": "1.4.2" + } + }, "hawk": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/hawk/-/hawk-6.0.2.tgz", @@ -198,6 +496,11 @@ "resolved": "https://registry.npmjs.org/hoek/-/hoek-4.2.1.tgz", "integrity": "sha512-QLg82fGkfnJ/4iy1xZ81/9SIJiq1NGFUMGs6ParyjBZr6jW2Ufj/snDqTHixNlHdPNwN2RLVD0Pi3igeK9+JfA==" }, + "http-cache-semantics": { + "version": "3.8.1", + "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz", + "integrity": "sha512-5ai2iksyV8ZXmnZhHH4rWPoxxistEexSi5936zIQ1bnNTW5VnA85B6P/VpXiRM017IgRvb2kKo1a//y+0wSp3w==" + }, "http-signature": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", @@ -208,22 +511,98 @@ "sshpk": "1.14.1" } }, + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + }, + "into-stream": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/into-stream/-/into-stream-3.1.0.tgz", + "integrity": "sha1-lvsKk2wSur1v8XUqF9BWFqvQlMY=", + "requires": { + "from2": "2.3.0", + "p-is-promise": "1.1.0" + } + }, + "is-callable": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.3.tgz", + "integrity": "sha1-hut1OSgF3cM69xySoO7fdO52BLI=" + }, + "is-date-object": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz", + "integrity": "sha1-mqIOtq7rv/d/vTPnTKAbM1gdOhY=" + }, + "is-object": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-object/-/is-object-1.0.1.tgz", + "integrity": "sha1-iVJojF7C/9awPsyF52ngKQMINHA=" + }, + "is-plain-obj": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz", + "integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4=" + }, + "is-regex": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz", + "integrity": "sha1-VRdIm1RwkbCTDglWVM7SXul+lJE=", + "requires": { + "has": "1.0.1" + } + }, + "is-retry-allowed": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-1.1.0.tgz", + "integrity": "sha1-EaBgVotnM5REAz0BJaYaINVk+zQ=" + }, + "is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=" + }, + "is-symbol": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.1.tgz", + "integrity": "sha1-PMWfAAJRlLarLjjbrmaJJWtmBXI=" + }, "is-typedarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, "isstream": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" }, + "isurl": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isurl/-/isurl-1.0.0.tgz", + "integrity": "sha512-1P/yWsxPlDtn7QeRD+ULKQPaIaN6yF368GZ2vDfv0AL0NwpStafjWCDDdn0k8wgFMWpVAqG7oJhxHnlud42i9w==", + "requires": { + "has-to-string-tag-x": "1.4.1", + "is-object": "1.0.1" + } + }, "jsbn": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=", "optional": true }, + "json-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.0.tgz", + "integrity": "sha1-Wx85evx11ne96Lz8Dkfh+aPZqJg=" + }, "json-schema": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", @@ -250,6 +629,24 @@ "verror": "1.10.0" } }, + "keyv": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-3.0.0.tgz", + "integrity": "sha512-eguHnq22OE3uVoSYG0LVWNP+4ppamWr9+zWBe1bsNcovIMy6huUJFPgy4mGwCd/rnl3vOLGW1MTlu4c57CT1xA==", + "requires": { + "json-buffer": "3.0.0" + } + }, + "loglevel": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.6.1.tgz", + "integrity": "sha1-4PyVEztu8nbNyIh82vJKpvFW+Po=" + }, + "lowercase-keys": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz", + "integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA==" + }, "mime-db": { "version": "1.33.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.33.0.tgz", @@ -263,21 +660,133 @@ "mime-db": "1.33.0" } }, + "mimic-response": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.0.tgz", + "integrity": "sha1-3z02Uqc/3ta5sLJBRub9BSNTRY4=" + }, "minimist": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" }, + "normalize-url": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-2.0.1.tgz", + "integrity": "sha512-D6MUW4K/VzoJ4rJ01JFKxDrtY1v9wrgzCX5f2qj/lzH1m/lW6MhUZFKerVsnyjOhOsYzI9Kqqak+10l4LvLpMw==", + "requires": { + "prepend-http": "2.0.0", + "query-string": "5.1.1", + "sort-keys": "2.0.0" + } + }, "oauth-sign": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz", "integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM=" }, + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" + }, + "object-keys": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.0.11.tgz", + "integrity": "sha1-xUYBd4rVYPEULODgG8yotW0TQm0=" + }, + "object.entries": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/object.entries/-/object.entries-1.0.4.tgz", + "integrity": "sha1-G/mk3SKI9bM/Opk9JXZh8F0WGl8=", + "requires": { + "define-properties": "1.1.2", + "es-abstract": "1.11.0", + "function-bind": "1.1.1", + "has": "1.0.1" + } + }, + "object.getownpropertydescriptors": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.0.3.tgz", + "integrity": "sha1-h1jIRvW0B62rDyNuCYbxSwUcqhY=", + "requires": { + "define-properties": "1.1.2", + "es-abstract": "1.11.0" + } + }, + "object.values": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.0.4.tgz", + "integrity": "sha1-5STaCbT2b/Bd9FdUbscqyZ8TBpo=", + "requires": { + "define-properties": "1.1.2", + "es-abstract": "1.11.0", + "function-bind": "1.1.1", + "has": "1.0.1" + } + }, + "p-cancelable": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-0.3.0.tgz", + "integrity": "sha512-RVbZPLso8+jFeq1MfNvgXtCRED2raz/dKpacfTNxsx6pLEpEomM7gah6VeHSYV3+vo0OAi4MkArtQcWWXuQoyw==" + }, + "p-defer": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-1.0.0.tgz", + "integrity": "sha1-n26xgvbJqozXQwBKfU+WsZaw+ww=" + }, + "p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=" + }, + "p-is-promise": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/p-is-promise/-/p-is-promise-1.1.0.tgz", + "integrity": "sha1-nJRWmJ6fZYgBewQ01WCXZ1w9oF4=" + }, + "p-queue": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-2.4.2.tgz", + "integrity": "sha512-n8/y+yDJwBjoLQe1GSJbbaYQLTI7QHNZI2+rpmCDbe++WLf9HC3gf6iqj5yfPAV71W4UF3ql5W1+UBPXoXTxng==" + }, + "p-retry": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-1.0.0.tgz", + "integrity": "sha1-OSczKkt9cCabU1UVEX/FR9oaaWg=", + "requires": { + "retry": "0.10.1" + } + }, + "p-timeout": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-2.0.1.tgz", + "integrity": "sha512-88em58dDVB/KzPEx1X0N3LwFfYZPyDc4B6eF38M1rk9VTZMbxXXgjugz8mmwpS9Ox4BDZ+t6t3QP5+/gazweIA==", + "requires": { + "p-finally": "1.0.0" + } + }, "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" }, + "pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=" + }, + "prepend-http": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-2.0.0.tgz", + "integrity": "sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc=" + }, + "process-nextick-args": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", + "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" + }, "punycode": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", @@ -288,6 +797,30 @@ "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz", "integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A==" }, + "query-string": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/query-string/-/query-string-5.1.1.tgz", + "integrity": "sha512-gjWOsm2SoGlgLEdAGt7a6slVOk9mGiXmPFMqrEhLQ68rhQuBnpfs3+EmlvqKyxnCo9/PPlF+9MtY02S1aFg+Jw==", + "requires": { + "decode-uri-component": "0.2.0", + "object-assign": "4.1.1", + "strict-uri-encode": "1.1.0" + } + }, + "readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "1.0.0", + "process-nextick-args": "2.0.0", + "safe-buffer": "5.1.1", + "string_decoder": "1.1.1", + "util-deprecate": "1.0.2" + } + }, "request": { "version": "2.85.0", "resolved": "https://registry.npmjs.org/request/-/request-2.85.0.tgz", @@ -317,6 +850,19 @@ "uuid": "3.2.1" } }, + "responselike": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/responselike/-/responselike-1.0.2.tgz", + "integrity": "sha1-kYcg7ztjHFZCvgaPFa3lpG9Loec=", + "requires": { + "lowercase-keys": "1.0.1" + } + }, + "retry": { + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.10.1.tgz", + "integrity": "sha1-52OI0heZLCUnUCQdPTlW/tmNj/Q=" + }, "safe-buffer": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", @@ -330,6 +876,14 @@ "hoek": "4.2.1" } }, + "sort-keys": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/sort-keys/-/sort-keys-2.0.0.tgz", + "integrity": "sha1-ZYU1WEhh7JfXMNbPQYIuH1ZoQSg=", + "requires": { + "is-plain-obj": "1.1.0" + } + }, "sshpk": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.14.1.tgz", @@ -345,11 +899,29 @@ "tweetnacl": "0.14.5" } }, + "strict-uri-encode": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz", + "integrity": "sha1-J5siXfHVgrH1TmWt3UNS4Y+qBxM=" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "5.1.1" + } + }, "stringstream": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz", "integrity": "sha1-TkhM1N5aC7vuGORjB3EKioFiGHg=" }, + "timed-out": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-4.0.1.tgz", + "integrity": "sha1-8y6srFoXW+ol1/q1Zas+2HQe9W8=" + }, "tough-cookie": { "version": "2.3.4", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.4.tgz", @@ -372,6 +944,29 @@ "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", "optional": true }, + "url-join": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/url-join/-/url-join-4.0.0.tgz", + "integrity": "sha1-TTNA6AfTdzvamZH4MFrNzCpmXSo=" + }, + "url-parse-lax": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-3.0.0.tgz", + "integrity": "sha1-FrXK/Afb42dsGxmZF3gj1lA6yww=", + "requires": { + "prepend-http": "2.0.0" + } + }, + "url-to-options": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/url-to-options/-/url-to-options-1.0.1.tgz", + "integrity": "sha1-FQWgOiiaSMvXpDTvuu7FBV9WM6k=" + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + }, "uuid": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.2.1.tgz", @@ -386,6 +981,15 @@ "core-util-is": "1.0.2", "extsprintf": "1.3.0" } + }, + "ws": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-4.1.0.tgz", + "integrity": "sha512-ZGh/8kF9rrRNffkLFV4AzhvooEclrOH0xaugmqGsIfFgOE/pIz4fMc4Ef+5HSQqTEug2S9JZIWDR47duDSLfaA==", + "requires": { + "async-limiter": "1.0.0", + "safe-buffer": "5.1.1" + } } } } diff --git a/package.json b/package.json index 838862f..886b950 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "license": "UNLICENSED", "private": true, "dependencies": { + "@slack/client": "^4.2.0", "dotenv": "^5.0.1", "minimist": "^1.2.0", "request": "^2.85.0" diff --git a/spawn.js b/spawn.js new file mode 100644 index 0000000..13ee81b --- /dev/null +++ b/spawn.js @@ -0,0 +1,173 @@ +require('dotenv').load(); + +const { RTMClient, WebClient } = require('@slack/client'); + +// An access token (from your Slack app or custom integration - usually xoxb) +const token = process.env.SLACK_CLI_TOKEN; + +class SpawnClient { + constructor(){ + console.log('signing in', token) + this.web = new WebClient(token); + this.rtm = new RTMClient(token) + this.rtm.on('connected', () => console.log('connected')) + this.rtm.on('authenticated', () => console.log('authenticated')) + this.rtm.start(); + + this.listen() + this.list() + } + listen(){ + console.log('listening') + this.rtm.on('message', (message) => { + console.log(message) + if ( (message.subtype && message.subtype === 'bot_message') || + (!message.subtype && message.user === this.rtm.activeUserId) ) { + return; + } + switch (message.subtype) { + case 'file_share': + this.processFile(message.file) + break + case 'file_comment': + this.processFileCommand(message.file, message.comment) + break + // case 'file_mention': + // break + default: + console.log(`(channel:${message.channel}) ${message.user} says: ${message.text}`); + break + } + }) + } + list(){ + console.log('listing groups..') + this.web.groups.list().then((res) => { + console.log("____________\n") + console.log(res) + + const group = res.groups[0] + + if (group) { + this.group = group + console.log('we are in', group) + } else { + console.log('This bot does not belong to any private group, invite it to at least one and try again'); + } + }); + } + + postMessageToGroup(msg){ + if (! this.group) { + console.log('not in a group!') + return + } + rtm.sendMessage(msg, this.group.id) + .then((msg) => console.log(`ts:${msg.ts} <${group.name}> ${msg}`)) + .catch(console.error); + } + + processFile(){ + console.log(message.text) + console.log(`download: ${message.file.url_private_download}`) + } + + processFileCommand(file, comment){ + const msg = comment.comment + console.log(`comment on file: ${msg}`) + } +} + +const client = new SpawnClient () + +// This argument can be a channel ID, a DM ID, a MPDM ID, or a group ID +// const conversationId = 'C1232456'; + +// The RTM client can send simple string messages +// rtm.sendMessage('Hello there', conversationId) +// .then((res) => { +// // `res` contains information about the posted message +// console.log('Message sent: ', res.ts); +// }) +// .catch(console.error); + + + +// { +// "type": "message", +// "subtype": "file_share", +// "ts": "1358877455.000010", +// "text": "<@cal> uploaded a file: ", +// "file": {...}, +// "user": "U2147483697", +// "upload": true +// } + +// { +// "type": "message", +// "subtype": "file_comment", +// "ts": "1361482916.000003", +// "text": "<@cal> commented on a file: ...", +// "file": {}, +// "comment": {} +// } + +// { +// "id" : "F2147483862", +// "created" : 1356032811, +// "timestamp" : 1356032811, +// "name" : "file.htm", +// "title" : "My HTML file", +// "mimetype" : "text\/plain", +// "filetype" : "text", +// "pretty_type": "Text", +// "user" : "U2147483697", +// "mode" : "hosted", +// "editable" : true, +// "is_external": false, +// "external_type": "", +// "username": "", +// "size" : 12345, +// "url_private": "https:\/\/slack.com\/files-pri\/T024BE7LD-F024BERPE\/1.png", +// "url_private_download": "https:\/\/slack.com\/files-pri\/T024BE7LD-F024BERPE\/download\/1.png", +// "thumb_64": "https:\/\/slack-files.com\/files-tmb\/T024BE7LD-F024BERPE-c66246\/1_64.png", +// "thumb_80": "https:\/\/slack-files.com\/files-tmb\/T024BE7LD-F024BERPE-c66246\/1_80.png", +// "thumb_360": "https:\/\/slack-files.com\/files-tmb\/T024BE7LD-F024BERPE-c66246\/1_360.png", +// "thumb_360_gif": "https:\/\/slack-files.com\/files-tmb\/T024BE7LD-F024BERPE-c66246\/1_360.gif", +// "thumb_360_w": 100, +// "thumb_360_h": 100, +// "thumb_480": "https:\/\/slack-files.com\/files-tmb\/T024BE7LD-F024BERPE-c66246\/1_480.png", +// "thumb_480_w": 480, +// "thumb_480_h": 480, +// "thumb_160": "https:\/\/slack-files.com\/files-tmb\/T024BE7LD-F024BERPE-c66246\/1_160.png", +// "permalink": "https:\/\/tinyspeck.slack.com\/files\/cal\/F024BERPE\/1.png", +// "permalink_public" : "https:\/\/tinyspeck.slack.com\/T024BE7LD-F024BERPE-3f9216b62c", +// "edit_link": "https:\/\/tinyspeck.slack.com\/files\/cal\/F024BERPE\/1.png/edit", +// "preview": "<!DOCTYPE html>\n<html>\n<meta charset='utf-8'>", +// "preview_highlight": "<div class=\"sssh-code\"><div class=\"sssh-line\"><pre><!DOCTYPE html...", +// "lines": 123, +// "lines_more": 118, +// "is_public": true, +// "public_url_shared": false, +// "display_as_bot" : false, +// "channels": ["C024BE7LT", ...], +// "groups": ["G12345", ...], +// "ims": ["D12345", ...], +// "initial_comment": {...}, +// "num_stars": 7, +// "is_starred": true, +// "pinned_to": ["C024BE7LT", ...], +// "reactions": [ +// { +// "name": "astonished", +// "count": 3, +// "users": [ "U1", "U2", "U3" ] +// }, +// { +// "name": "facepalm", +// "count": 1034, +// "users": [ "U1", "U2", "U3", "U4", "U5" ] +// } +// ], +// "comments_count": 1 +// } \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 0f2cf9df81a6a39aa25be3cca828c4efa47c53a3 Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Sat, 12 May 2018 04:25:55 +0200 Subject: fns --- process_fns | 2 +- upload.js | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/process_fns b/process_fns index ad47a28..3964b3b 100755 --- a/process_fns +++ b/process_fns @@ -1,4 +1,4 @@ -workon samplernn +# workon samplernn function generate () { exp_name=$1 diff --git a/upload.js b/upload.js index 73e1147..1abc5e9 100755 --- a/upload.js +++ b/upload.js @@ -32,6 +32,7 @@ var r = request.post('https://bucky.asdf.us/api/thread/' + id + '/comment', func console.log('\n') console.log('Upload successful!') console.log('\n') + console.log(body) argv['_'].forEach(fn => { const partz = fn.split('/') const url = 'https://s3.amazonaws.com/i.asdf.us/bucky/data/' + id + '/' + partz[partz.length-1] -- cgit v1.2.3-70-g09d2 From 3ff94cfea6b8953307e29d5d3ffba9d70cf7659e Mon Sep 17 00:00:00 2001 From: Jules Laplace Date: Sat, 12 May 2018 04:28:01 +0200 Subject: gen len --- process_fns | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/process_fns b/process_fns index 3964b3b..0bac4e7 100755 --- a/process_fns +++ b/process_fns @@ -32,6 +32,37 @@ function generate () { --resume True sleep 1 } +function gen_len () { + exp_name=$1 + n_samples=$2 + sample_rate=$3 + duration=$4 + sample_length=$5 + + echo "" + echo "###################################################" + echo "###################################################" + echo "###################################################" + echo "" + echo ">> generating $exp_name" + echo "" + echo "###################################################" + echo "###################################################" + echo "###################################################" + echo "" + python generate.py \ + --exp $exp_name --dataset $exp_name \ + --frame_sizes 8 2 \ + --n_rnn 2 --dim 1024 --q_levels 256 \ + --seq_len 1024 --batch_size 128 \ + --val_frac 0.1 --test_frac 0.1 \ + --sample_rate $sample_rate \ + --sample_length $sample_length \ + --keep_old_checkpoints False \ + --n_samples $n_samples \ + --epoch_limit 1 \ + --resume True +} function run () { exp_name=$1 epoch_limit=$2 -- cgit v1.2.3-70-g09d2 From a61412eb1538615578e87a20ddfbc618e1eeb041 Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Sun, 13 May 2018 12:03:37 +0200 Subject: get script now supports urls --- get.pl | 43 ++++++++++++++++++++++++++++++++++++++----- latest.pl | 7 ++++++- process_fns | 3 +-- 3 files changed, 45 insertions(+), 8 deletions(-) diff --git a/get.pl b/get.pl index 4447ab3..8a0d8f9 100755 --- a/get.pl +++ b/get.pl @@ -4,11 +4,44 @@ $SIG{TERM} = $SIG{INT} = sub { exit 1 }; chdir('datasets'); -foreach (@ARGV){ - my $s = $_; - if ($s !~/\..*$/) { $s .= ".wav"; } - system('/usr/bin/wget', 'https://neural:spawn5@asdf.us/neural/' . $s); - system('/usr/bin/perl', 'dataset.pl', $s); +my $fn, $new_fn; +foreach my $s (@ARGV){ + if ($s =~ /^http/) { + if ($s =~ /(wav|aiff?|flac|mp3|opus)$/i) { + my $fn = `basename $s`; + system('/usr/bin/wget', $s); + system('/usr/bin/perl', 'dataset.pl', $fn); + } else { + my $yt = `youtube-dl --extract-audio --audio-format flac -o "%(title)s.%(ext)s" $s`; + my @partz = split("\n", $yt); + foreach $part (@partz) { + if ($part =~ /\[ffmpeg\] Destination\: (.*\.flac)$/) { + $fn = $1; + } + } + if ($fn) { + $new_fn = lc $fn; + $new_fn =~ s/\.flac$//g; + $new_fn =~ s/\s+/_/g; + $new_fn =~ s/\W//g; + if (length($new_fn) == 0) { + $new_fn = lc $s; + $new_fn =~ s/\s+/_/g; + $new_fn =~ s/\W//g; + } + $new_fn =~ s/_+/_/g; + $new_fn .= '.flac'; + system('mv', $fn, $new_fn); + print"got fn, $fn => $new_fn\n"; + system('/usr/bin/perl', 'dataset.pl', $new_fn); + } + } + } else { + if ($s !~/\..*$/) { $s .= ".wav"; } + system('/usr/bin/wget', 'https://neural:spawn5@asdf.us/neural/' . $s); + system('/usr/bin/perl', 'dataset.pl', $s); + } + $fn = undef; } END { diff --git a/latest.pl b/latest.pl index ddd5952..1bb7932 100755 --- a/latest.pl +++ b/latest.pl @@ -5,7 +5,8 @@ use Getopt::Std; our $opt_l; # mix and upload files our $opt_v; # print log for all files -getopts('lav'); +our $opt_n; # name/tag for file +getopts('lavn:'); our $matches = 0; if (scalar @ARGV) { @@ -73,6 +74,10 @@ sub process($){ # return 0; #} + if ($opt_l && $opt_n) { + $name .= '-' . $opt_n; + } + if (-e "output/" . $name . ".mp3") { return 1; } diff --git a/process_fns b/process_fns index 0bac4e7..0cbdeee 100755 --- a/process_fns +++ b/process_fns @@ -36,8 +36,7 @@ function gen_len () { exp_name=$1 n_samples=$2 sample_rate=$3 - duration=$4 - sample_length=$5 + sample_length=$4 echo "" echo "###################################################" -- cgit v1.2.3-70-g09d2 From e2d8a6f26c5e44d970d7c069f171105376835495 Mon Sep 17 00:00:00 2001 From: jules on spawn Date: Mon, 14 May 2018 18:57:17 +0200 Subject: log stuff.. auto append --- get.pl | 7 +++++++ process_fns | 11 ++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/get.pl b/get.pl index 8a0d8f9..0ef39db 100755 --- a/get.pl +++ b/get.pl @@ -9,9 +9,11 @@ foreach my $s (@ARGV){ if ($s =~ /^http/) { if ($s =~ /(wav|aiff?|flac|mp3|opus)$/i) { my $fn = `basename $s`; + print "downloading $fn\n"; system('/usr/bin/wget', $s); system('/usr/bin/perl', 'dataset.pl', $fn); } else { + print "youtube-dl $s\n"; my $yt = `youtube-dl --extract-audio --audio-format flac -o "%(title)s.%(ext)s" $s`; my @partz = split("\n", $yt); foreach $part (@partz) { @@ -38,9 +40,14 @@ foreach my $s (@ARGV){ } } else { if ($s !~/\..*$/) { $s .= ".wav"; } + print "downloading $s\n"; system('/usr/bin/wget', 'https://neural:spawn5@asdf.us/neural/' . $s); system('/usr/bin/perl', 'dataset.pl', $s); } + open(my $fd, ">>../run_slap.sh"); + print $fd "standard $fn"; + close $fn; + $fn = undef; } diff --git a/process_fns b/process_fns index 0cbdeee..cef738d 100755 --- a/process_fns +++ b/process_fns @@ -1,12 +1,13 @@ # workon samplernn function generate () { - exp_name=$1 + exp_name=${1%.*} n_samples=$2 sample_rate=$3 duration=$4 let sample_length=$3*$4 + sleep 0.1 echo "" echo "###################################################" echo "###################################################" @@ -30,14 +31,14 @@ function generate () { --n_samples $n_samples \ --epoch_limit 1 \ --resume True - sleep 1 } function gen_len () { - exp_name=$1 + exp_name=${1%.*} n_samples=$2 sample_rate=$3 sample_length=$4 + sleep 0.1 echo "" echo "###################################################" echo "###################################################" @@ -63,13 +64,14 @@ function gen_len () { --resume True } function run () { - exp_name=$1 + exp_name=${1%.*} epoch_limit=$2 n_samples=$3 sample_rate=$4 duration=$5 let sample_length=$4*$5 + sleep 0.1 echo "" echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" @@ -93,7 +95,6 @@ function run () { --keep_old_checkpoints False \ --epoch_limit $epoch_limit \ --resume True - sleep 1 } function standard () { dataset=$1 -- cgit v1.2.3-70-g09d2