diff options
| -rw-r--r-- | become_yukarin/data_struct.py | 64 | ||||
| -rw-r--r-- | become_yukarin/voice_changer.py | 219 | ||||
| -rw-r--r-- | scripts/realtime_voice_changer.py | 153 | ||||
| -rw-r--r-- | tests/test_voice_changer.py | 38 |
4 files changed, 355 insertions, 119 deletions
diff --git a/become_yukarin/data_struct.py b/become_yukarin/data_struct.py index 78c8cf3..4474331 100644 --- a/become_yukarin/data_struct.py +++ b/become_yukarin/data_struct.py @@ -1,8 +1,9 @@ -from typing import NamedTuple +from typing import NamedTuple, Dict, List +import numpy import pyworld -import numpy +_min_mc = -18.3 class Wave(NamedTuple): @@ -11,11 +12,21 @@ class Wave(NamedTuple): class AcousticFeature(NamedTuple): - f0: numpy.ndarray - spectrogram: numpy.ndarray - aperiodicity: numpy.ndarray - mfcc: numpy.ndarray - voiced: numpy.ndarray + f0: numpy.ndarray = numpy.nan + spectrogram: numpy.ndarray = numpy.nan + aperiodicity: numpy.ndarray = numpy.nan + mfcc: numpy.ndarray = numpy.nan + voiced: numpy.ndarray = numpy.nan + + @staticmethod + def dtypes(): + return dict( + f0=numpy.float32, + spectrogram=numpy.float32, + aperiodicity=numpy.float32, + mfcc=numpy.float32, + voiced=numpy.bool, + ) def astype(self, dtype): return AcousticFeature( @@ -51,6 +62,45 @@ class AcousticFeature(NamedTuple): assert self.voiced.dtype == numpy.bool @staticmethod + def silent(length: int, sizes: Dict[str, int], keys: List[str]): + d = {} + if 'f0' in keys: + d['f0'] = numpy.zeros((length, sizes['f0']), dtype=AcousticFeature.dtypes()['f0']) + if 'spectrogram' in keys: + d['spectrogram'] = numpy.zeros((length, sizes['spectrogram']), + dtype=AcousticFeature.dtypes()['spectrogram']) + if 'aperiodicity' in keys: + d['aperiodicity'] = numpy.zeros((length, sizes['aperiodicity']), + dtype=AcousticFeature.dtypes()['aperiodicity']) + if 'mfcc' in keys: + d['mfcc'] = numpy.hstack(( + numpy.ones((length, 1), dtype=AcousticFeature.dtypes()['mfcc']) * _min_mc, + numpy.zeros((length, sizes['mfcc'] - 1), dtype=AcousticFeature.dtypes()['mfcc']) + )) + if 'voiced' in keys: + d['voiced'] = numpy.zeros((length, sizes['voiced']), dtype=AcousticFeature.dtypes()['voiced']) + feature = AcousticFeature(**d) + return feature + + @staticmethod + def concatenate(fs: List['AcousticFeature'], keys: List[str]): + is_target = lambda a: not numpy.any(numpy.isnan(a)) + return AcousticFeature(**{ + key: numpy.concatenate([getattr(f, key) for f in fs]) if is_target(getattr(fs[0], key)) else numpy.nan + for key in keys + }) + + def pick(self, first: int, last: int): + is_target = lambda a: not numpy.any(numpy.isnan(a)) + return AcousticFeature( + f0=self.f0[first:last] if is_target(self.f0) else numpy.nan, + spectrogram=self.spectrogram[first:last] if is_target(self.spectrogram) else numpy.nan, + aperiodicity=self.aperiodicity[first:last] if is_target(self.aperiodicity) else numpy.nan, + mfcc=self.mfcc[first:last] if is_target(self.mfcc) else numpy.nan, + voiced=self.voiced[first:last] if is_target(self.voiced) else numpy.nan, + ) + + @staticmethod def get_sizes(sampling_rate: int, order: int): fft_size = pyworld.get_cheaptrick_fft_size(fs=sampling_rate) return dict( diff --git a/become_yukarin/voice_changer.py b/become_yukarin/voice_changer.py index 5e0eac0..bed155f 100644 --- a/become_yukarin/voice_changer.py +++ b/become_yukarin/voice_changer.py @@ -1,4 +1,5 @@ -from typing import List +from abc import ABCMeta, abstractproperty, abstractmethod +from typing import List, Callable, Any from typing import NamedTuple import numpy @@ -15,7 +16,6 @@ class VoiceChanger(object): self, acoustic_converter: AcousticConverter, super_resolution: SuperResolution, - vocoder: Vocoder, output_sampling_rate: int = None, ) -> None: if output_sampling_rate is None: @@ -23,19 +23,8 @@ class VoiceChanger(object): self.acoustic_converter = acoustic_converter self.super_resolution = super_resolution - self.vocoder = vocoder self.output_sampling_rate = output_sampling_rate - def convert_from_wave_path(self, wave_path: str): - w_in = self.acoustic_converter._wave_process(wave_path) - return self.convert_from_wave(w_in) - - def convert_from_wave(self, wave: Wave): - f_in = self.acoustic_converter._feature_process(wave) - f_high = self.convert_from_acoustic_feature(f_in) - wave = self.vocoder.decode(f_high) - return wave - def convert_from_acoustic_feature(self, f_in: AcousticFeature): f_low = self.acoustic_converter.convert_to_feature(f_in) s_high = self.super_resolution.convert(f_low.spectrogram.astype(numpy.float32)) @@ -43,7 +32,35 @@ class VoiceChanger(object): return f_high -class Segment(NamedTuple): +class BaseSegment(ABCMeta): + start_time: float + + @property + @abstractmethod + def time_length(self) -> float: + pass + + @property + @abstractmethod + def end_time(self) -> float: + pass + + +class FeatureSegment(NamedTuple, BaseSegment): + start_time: float + feature: AcousticFeature + frame_period: float + + @property + def time_length(self): + return len(self.feature.f0) * self.frame_period / 1000 + + @property + def end_time(self): + return self.time_length + self.start_time + + +class WaveSegment(NamedTuple, BaseSegment): start_time: float wave: Wave @@ -59,44 +76,73 @@ class Segment(NamedTuple): class VoiceChangerStream(object): def __init__( self, - voice_changer: VoiceChanger, sampling_rate: int, + frame_period: float, in_dtype=numpy.float32, ): - self.voice_changer = voice_changer self.sampling_rate = sampling_rate + self.frame_period = frame_period self.in_dtype = in_dtype - self._data_stream = [] # type: List[Segment] - @property - def vocoder(self): - return self.voice_changer.vocoder + self.voice_changer: VoiceChanger = None + self.vocoder: Vocoder = None + self._data_stream = [] # type: List[WaveSegment] + self._in_feature_stream = [] # type: List[FeatureSegment] + self._out_feature_stream = [] # type: List[FeatureSegment] def add_wave(self, start_time: float, wave: Wave): # validation assert wave.sampling_rate == self.sampling_rate assert wave.wave.dtype == self.in_dtype - segment = Segment(start_time=start_time, wave=wave) + segment = WaveSegment(start_time=start_time, wave=wave) self._data_stream.append(segment) - def remove_wave(self, end_time: float): + def add_in_feature(self, start_time: float, feature: AcousticFeature, frame_period: float): + # validation + assert frame_period == self.frame_period + assert feature.f0.dtype == self.in_dtype + + segment = FeatureSegment(start_time=start_time, feature=feature, frame_period=self.frame_period) + self._in_feature_stream.append(segment) + + def add_out_feature(self, start_time: float, feature: AcousticFeature, frame_period: float): + # validation + assert frame_period == self.frame_period + + segment = FeatureSegment(start_time=start_time, feature=feature, frame_period=self.frame_period) + self._out_feature_stream.append(segment) + + def remove(self, end_time: float): self._data_stream = list(filter(lambda s: s.end_time > end_time, self._data_stream)) + self._in_feature_stream = list(filter(lambda s: s.end_time > end_time, self._in_feature_stream)) + self._out_feature_stream = list(filter(lambda s: s.end_time > end_time, self._out_feature_stream)) + + @staticmethod + def fetch( + start_time: float, + time_length: float, + data_stream: List[BaseSegment], + rate: float, + pad_function: Callable[[int], Any], + pick_function: Callable[[Any, int, int], Any], + concat_function: Callable[[List], Any], + extra_time: float = 0, + ): + start_time -= extra_time + time_length += extra_time * 2 - def convert_to_feature(self, start_time: float, time_length: float): end_time = start_time + time_length buffer_list = [] - stream = filter(lambda s: not (end_time < s.start_time or s.end_time < start_time), self._data_stream) + stream = filter(lambda s: not (end_time < s.start_time or s.end_time < start_time), data_stream) start_time_buffer = start_time remaining_time = time_length for segment in stream: # padding if segment.start_time > start_time_buffer: - pad = numpy.zeros( - shape=int((segment.start_time - start_time_buffer) * self.sampling_rate), - dtype=self.in_dtype, - ) + length = int((segment.start_time - start_time_buffer) * rate) + pad = pad_function(length) buffer_list.append(pad) start_time_buffer = segment.start_time @@ -105,9 +151,9 @@ class VoiceChangerStream(object): else: one_time_length = remaining_time - first_index = int((start_time_buffer - segment.start_time) * self.sampling_rate) - last_index = int(first_index + one_time_length * self.sampling_rate) - one_buffer = segment.wave.wave[first_index:last_index] + first_index = int((start_time_buffer - segment.start_time) * rate) + last_index = int(first_index + one_time_length * rate) + one_buffer = pick_function(segment, first_index, last_index) buffer_list.append(one_buffer) start_time_buffer += one_time_length @@ -117,44 +163,67 @@ class VoiceChangerStream(object): break else: # last padding - pad = numpy.zeros(shape=int((end_time - start_time_buffer) * self.sampling_rate), dtype=self.in_dtype) + length = int((end_time - start_time_buffer) * rate) + pad = pad_function(length) buffer_list.append(pad) - buffer = numpy.concatenate(buffer_list) - in_wave = Wave(wave=buffer, sampling_rate=self.sampling_rate) - in_feature = self.vocoder.encode(in_wave) - out_feature = self.voice_changer.convert_from_acoustic_feature(in_feature) - return out_feature + buffer = concat_function(buffer_list) + return buffer - def convert(self, start_time: float, time_length: float): - feature = self.convert_to_feature(start_time=start_time, time_length=time_length) - out_wave = self.vocoder.decode( - acoustic_feature=feature, + def pre_convert(self, start_time: float, time_length: float, extra_time: float): + wave = self.fetch( + start_time=start_time, + time_length=time_length, + extra_time=extra_time, + data_stream=self._data_stream, + rate=self.sampling_rate, + pad_function=lambda length: numpy.zeros(shape=length, dtype=self.in_dtype), + pick_function=lambda segment, first, last: segment.wave.wave[first:last], + concat_function=numpy.concatenate, ) - return out_wave + in_wave = Wave(wave=wave, sampling_rate=self.sampling_rate) + in_feature = self.vocoder.encode(in_wave) - def convert_with_extra_time(self, start_time: float, time_length: float, extra_time: float): - """ - :param extra_time: 音声変換時に余分に使うデータの時間長。ゼロパディングを防ぐ。 - """ - frame_period = self.vocoder.acoustic_feature_param.frame_period + pad = int(extra_time / (self.vocoder.acoustic_feature_param.frame_period / 1000)) + in_feature = in_feature.pick(pad, -pad) + return in_feature - start_time -= extra_time - time_length += extra_time * 2 + def convert(self, start_time: float, time_length: float, extra_time: float): + order = self.voice_changer.acoustic_converter.config.dataset.param.acoustic_feature_param.order + sizes = AcousticFeature.get_sizes(sampling_rate=self.sampling_rate, order=order) + keys = ['f0', 'aperiodicity', 'mfcc', 'voiced'] + in_feature = self.fetch( + start_time=start_time, + time_length=time_length, + extra_time=extra_time, + data_stream=self._in_feature_stream, + rate=1000 / self.frame_period, + pad_function=lambda length: AcousticFeature.silent(length, sizes=sizes, keys=keys), + pick_function=lambda segment, first, last: segment.feature.pick(first, last), + concat_function=lambda buffers: AcousticFeature.concatenate(buffers, keys=keys), + ) + out_feature = self.voice_changer.convert_from_acoustic_feature(in_feature) - extra_feature = self.convert_to_feature(start_time=start_time, time_length=time_length) + pad = int(extra_time * 1000 / self.frame_period) + out_feature = out_feature.pick(pad, -pad) + return out_feature - pad = int(extra_time / (frame_period / 1000)) - feature = AcousticFeature( - f0=extra_feature.f0[pad:-pad], - spectrogram=extra_feature.spectrogram[pad:-pad], - aperiodicity=extra_feature.aperiodicity[pad:-pad], - mfcc=extra_feature.mfcc[pad:-pad], - voiced=extra_feature.voiced[pad:-pad], + def post_convert(self, start_time: float, time_length: float): + order = self.voice_changer.acoustic_converter.config.dataset.param.acoustic_feature_param.order + sizes = AcousticFeature.get_sizes(sampling_rate=self.sampling_rate, order=order) + keys = ['f0', 'aperiodicity', 'spectrogram', 'voiced'] + out_feature = self.fetch( + start_time=start_time, + time_length=time_length, + data_stream=self._out_feature_stream, + rate=1000 / self.frame_period, + pad_function=lambda length: AcousticFeature.silent(length, sizes=sizes, keys=keys), + pick_function=lambda segment, first, last: segment.feature.pick(first, last), + concat_function=lambda buffers: AcousticFeature.concatenate(buffers, keys=keys), ) out_wave = self.vocoder.decode( - acoustic_feature=feature, + acoustic_feature=out_feature, ) return out_wave @@ -163,20 +232,46 @@ class VoiceChangerStreamWrapper(object): def __init__( self, voice_changer_stream: VoiceChangerStream, - extra_time: float = 0.0 + extra_time_pre: float = 0.0, + extra_time: float = 0.0, ): self.voice_changer_stream = voice_changer_stream + self.extra_time_pre = extra_time_pre self.extra_time = extra_time + self._current_time_pre = 0 self._current_time = 0 + self._current_time_post = 0 + + def pre_convert_next(self, time_length: float): + in_feature = self.voice_changer_stream.pre_convert( + start_time=self._current_time_pre, + time_length=time_length, + extra_time=self.extra_time_pre, + ) + self._current_time_pre += time_length + return in_feature def convert_next(self, time_length: float): - out_wave = self.voice_changer_stream.convert_with_extra_time( + out_feature = self.voice_changer_stream.convert( start_time=self._current_time, time_length=time_length, extra_time=self.extra_time, ) self._current_time += time_length + return out_feature + + def post_convert_next(self, time_length: float): + out_wave = self.voice_changer_stream.post_convert( + start_time=self._current_time_post, + time_length=time_length, + ) + self._current_time_post += time_length return out_wave - def remove_previous_wave(self): - self.voice_changer_stream.remove_wave(end_time=self._current_time - self.extra_time) + def remove_previous(self): + end_time = min( + self._current_time_pre - self.extra_time_pre, + self._current_time - self.extra_time, + self._current_time_post, + ) + self.voice_changer_stream.remove(end_time=end_time) diff --git a/scripts/realtime_voice_changer.py b/scripts/realtime_voice_changer.py index a5d1a21..e96ce4e 100644 --- a/scripts/realtime_voice_changer.py +++ b/scripts/realtime_voice_changer.py @@ -14,92 +14,128 @@ import numpy import pyaudio from become_yukarin import AcousticConverter +from become_yukarin import Vocoder from become_yukarin import RealtimeVocoder from become_yukarin import SuperResolution from become_yukarin import VoiceChanger +from become_yukarin.config.config import Config from become_yukarin.config.config import create_from_json as create_config from become_yukarin.config.sr_config import create_from_json as create_sr_config from become_yukarin.data_struct import Wave +from become_yukarin.data_struct import AcousticFeature from become_yukarin.voice_changer import VoiceChangerStream from become_yukarin.voice_changer import VoiceChangerStreamWrapper class AudioConfig(NamedTuple): rate: int + frame_period: float audio_chunk: int convert_chunk: int vocoder_buffer_size: int out_norm: float -def convert_worker( - config, - acoustic_converter, - super_resolution, +def encode_worker( + config: Config, + wrapper: VoiceChangerStreamWrapper, audio_config: AudioConfig, - queue_input_wave, - queue_output_wave, + queue_input: Queue, + queue_output: Queue, ): - vocoder = RealtimeVocoder( + wrapper.voice_changer_stream.vocoder = Vocoder( acoustic_feature_param=config.dataset.param.acoustic_feature_param, out_sampling_rate=audio_config.rate, - buffer_size=audio_config.vocoder_buffer_size, - number_of_pointers=16, ) - # vocoder.warm_up(audio_config.vocoder_buffer_size / config.dataset.param.voice_param.sample_rate) - voice_changer = VoiceChanger( + start_time = 0 + time_length = audio_config.convert_chunk / audio_config.rate + + while True: + wave = queue_input.get() + + w = Wave(wave=wave, sampling_rate=audio_config.rate) + wrapper.voice_changer_stream.add_wave(start_time=start_time, wave=w) + start_time += time_length + + feature = wrapper.pre_convert_next(time_length=time_length) + queue_output.put(feature) + + +def convert_worker( + config: Config, + wrapper: VoiceChangerStreamWrapper, + acoustic_converter: AcousticConverter, + super_resolution: SuperResolution, + audio_config: AudioConfig, + queue_input: Queue, + queue_output: Queue, +): + wrapper.voice_changer_stream.voice_changer = VoiceChanger( super_resolution=super_resolution, acoustic_converter=acoustic_converter, - vocoder=vocoder, ) - voice_changer_stream = VoiceChangerStream( - voice_changer=voice_changer, - sampling_rate=audio_config.rate, - in_dtype=numpy.float32, - ) + start_time = 0 + time_length = audio_config.convert_chunk / audio_config.rate + while True: + in_feature: AcousticFeature = queue_input.get() + wrapper.voice_changer_stream.add_in_feature( + start_time=start_time, + feature=in_feature, + frame_period=audio_config.frame_period, + ) + start_time += time_length - wrapper = VoiceChangerStreamWrapper( - voice_changer_stream=voice_changer_stream, - extra_time=0.1, + out_feature = wrapper.convert_next(time_length=time_length) + queue_output.put(out_feature) + + +def decode_worker( + config: Config, + wrapper: VoiceChangerStreamWrapper, + audio_config: AudioConfig, + queue_input: Queue, + queue_output: Queue, +): + wrapper.voice_changer_stream.vocoder = RealtimeVocoder( + acoustic_feature_param=config.dataset.param.acoustic_feature_param, + out_sampling_rate=audio_config.rate, + buffer_size=audio_config.vocoder_buffer_size, + number_of_pointers=16, ) + # vocoder.warm_up(audio_config.vocoder_buffer_size / config.dataset.param.voice_param.sample_rate) start_time = 0 - wave = numpy.zeros(audio_config.convert_chunk * 2, dtype=numpy.float32) - wave = Wave(wave=wave, sampling_rate=audio_config.rate) - wrapper.voice_changer_stream.add_wave(start_time=start_time, wave=wave) - start_time += len(wave.wave) / wave.sampling_rate - wave = wrapper.convert_next(time_length=1) - time_length = audio_config.convert_chunk / audio_config.rate wave_fragment = numpy.empty(0) while True: - wave = queue_input_wave.get() - w = Wave(wave=wave, sampling_rate=audio_config.rate) - wrapper.voice_changer_stream.add_wave(start_time=start_time, wave=w) + feature: AcousticFeature = queue_input.get() + wrapper.voice_changer_stream.add_out_feature( + start_time=start_time, + feature=feature, + frame_period=audio_config.frame_period, + ) start_time += time_length - b = time.time() - wave = wrapper.convert_next(time_length=time_length).wave - print('time', time.time()-b, flush=True) - wrapper.remove_previous_wave() - print('converted wave', len(wave), flush=True) + wave = wrapper.post_convert_next(time_length=time_length).wave wave_fragment = numpy.concatenate([wave_fragment, wave]) if len(wave_fragment) >= audio_config.audio_chunk: wave, wave_fragment = wave_fragment[:audio_config.audio_chunk], wave_fragment[audio_config.audio_chunk:] - queue_output_wave.put(wave) + queue_output.put(wave) def main(): print('model loading...', flush=True) queue_input_wave = Queue() + queue_input_feature = Queue() + queue_output_feature = Queue() queue_output_wave = Queue() - model_path = Path('./trained/harvest-innoise03/predictor_1390000.npz') - config_path = Path('./trained/harvest-innoise03/config.json') + model_path = Path('./trained/pp-weakD-innoise01-tarnoise001/predictor_120000.npz') + config_path = Path('./trained/pp-weakD-innoise01-tarnoise001/config.json') config = create_config(config_path) acoustic_converter = AcousticConverter(config, model_path, gpu=0) print('model 1 loaded!', flush=True) @@ -113,23 +149,53 @@ def main(): audio_instance = pyaudio.PyAudio() audio_config = AudioConfig( rate=config.dataset.param.voice_param.sample_rate, + frame_period=config.dataset.param.acoustic_feature_param.frame_period, audio_chunk=config.dataset.param.voice_param.sample_rate, convert_chunk=config.dataset.param.voice_param.sample_rate, vocoder_buffer_size=config.dataset.param.voice_param.sample_rate // 16, out_norm=2.5, ) - process_converter = Process(target=convert_worker, kwargs=dict( + voice_changer_stream = VoiceChangerStream( + sampling_rate=audio_config.rate, + frame_period=config.dataset.param.acoustic_feature_param.frame_period, + in_dtype=numpy.float32, + ) + + wrapper = VoiceChangerStreamWrapper( + voice_changer_stream=voice_changer_stream, + extra_time_pre=0.2, + extra_time=0.1, + ) + + process_encoder = Process(target=encode_worker, kwargs=dict( config=config, + wrapper=wrapper, audio_config=audio_config, + queue_input=queue_input_wave, + queue_output=queue_input_feature, + )) + process_encoder.start() + + process_converter = Process(target=convert_worker, kwargs=dict( + config=config, + wrapper=wrapper, acoustic_converter=acoustic_converter, super_resolution=super_resolution, - queue_input_wave=queue_input_wave, - queue_output_wave=queue_output_wave, + audio_config=audio_config, + queue_input=queue_input_feature, + queue_output=queue_output_feature, )) process_converter.start() - signal.signal(signal.SIGINT, lambda signum, frame: process_converter.terminate()) + process_decoder = Process(target=decode_worker, kwargs=dict( + config=config, + wrapper=wrapper, + audio_config=audio_config, + queue_input=queue_output_feature, + queue_output=queue_output_wave, + )) + process_decoder.start() audio_stream = audio_instance.open( format=pyaudio.paFloat32, @@ -149,6 +215,11 @@ def main(): print('input', len(wave), flush=True) queue_input_wave.put(wave) + print('queue_input_wave', queue_input_wave.qsize(), flush=True) + print('queue_input_feature', queue_input_feature.qsize(), flush=True) + print('queue_output_feature', queue_output_feature.qsize(), flush=True) + print('queue_output_wave', queue_output_wave.qsize(), flush=True) + # output try: wave = queue_output_wave.get_nowait() diff --git a/tests/test_voice_changer.py b/tests/test_voice_changer.py index ceddf9c..66ea003 100644 --- a/tests/test_voice_changer.py +++ b/tests/test_voice_changer.py @@ -32,16 +32,16 @@ test_output_path = Path('output.wav') print('model loading...', flush=True) -model_path = model_base_path / Path('harvest-innoise03/predictor_1390000.npz') -config_path = model_base_path / Path('harvest-innoise03/config.json') +model_path = model_base_path / Path('pp-weakD-innoise01-tarnoise001/predictor_120000.npz') +config_path = model_base_path / Path('pp-weakD-innoise01-tarnoise001/config.json') config = create_config(config_path) -acoustic_converter = AcousticConverter(config, model_path, gpu=0) +acoustic_converter = AcousticConverter(config, model_path) print('model 1 loaded!', flush=True) model_path = model_base_path / Path('sr-noise3/predictor_180000.npz') config_path = model_base_path / Path('sr-noise3/config.json') sr_config = create_sr_config(config_path) -super_resolution = SuperResolution(sr_config, model_path, gpu=0) +super_resolution = SuperResolution(sr_config, model_path) print('model 2 loaded!', flush=True) audio_config = AudioConfig( @@ -50,6 +50,7 @@ audio_config = AudioConfig( vocoder_buffer_size=config.dataset.param.voice_param.sample_rate // 16, out_norm=4.5, ) +frame_period = config.dataset.param.acoustic_feature_param.frame_period vocoder = RealtimeVocoder( acoustic_feature_param=config.dataset.param.acoustic_feature_param, @@ -57,22 +58,24 @@ vocoder = RealtimeVocoder( buffer_size=audio_config.vocoder_buffer_size, number_of_pointers=16, ) -# vocoder.warm_up(audio_config.vocoder_buffer_size / config.dataset.param.voice_param.sample_rate) voice_changer = VoiceChanger( super_resolution=super_resolution, acoustic_converter=acoustic_converter, - vocoder=vocoder, ) voice_changer_stream = VoiceChangerStream( - voice_changer=voice_changer, sampling_rate=audio_config.rate, + frame_period=acoustic_converter._param.acoustic_feature_param.frame_period, in_dtype=numpy.float32, ) +voice_changer_stream.voice_changer = voice_changer +voice_changer_stream.vocoder = vocoder + wrapper = VoiceChangerStreamWrapper( voice_changer_stream=voice_changer_stream, + extra_time_pre=1, extra_time=0.2, ) @@ -85,9 +88,26 @@ for i in range(0, len(raw_wave), audio_config.chunk): wrapper.voice_changer_stream.add_wave(start_time=start_time, wave=wave_in) start_time += len(wave_in.wave) / wave_in.sampling_rate - wave_out = wrapper.convert_next(time_length=audio_config.chunk / audio_config.rate) +start_time = 0 +for i in range(len(raw_wave) // audio_config.chunk + 1): + feature_in = wrapper.pre_convert_next(time_length=audio_config.chunk / audio_config.rate) + wrapper.voice_changer_stream.add_in_feature(start_time=start_time, feature=feature_in, frame_period=frame_period) + start_time += audio_config.chunk / audio_config.rate + print('pre', i, flush=True) + +start_time = 0 +for i in range(len(raw_wave) // audio_config.chunk + 1): + feature_out = wrapper.convert_next(time_length=audio_config.chunk / audio_config.rate) + wrapper.voice_changer_stream.add_out_feature(start_time=start_time, feature=feature_out, frame_period=frame_period) + start_time += audio_config.chunk / audio_config.rate + print('cent', i, flush=True) + +start_time = 0 +for i in range(len(raw_wave) // audio_config.chunk + 1): + wave_out = wrapper.post_convert_next(time_length=audio_config.chunk / audio_config.rate) wave_out_list.append(wave_out) - wrapper.remove_previous_wave() + start_time += audio_config.chunk / audio_config.rate + print('post', i, flush=True) out_wave = numpy.concatenate([w.wave for w in wave_out_list]).astype(numpy.float32) librosa.output.write_wav(str(test_output_path), out_wave, sr=audio_config.rate) |