Spaces:
Build error
Build error
| import torch | |
| import numpy as np | |
| import sys | |
| import torch.nn.functional as torch_nn_func | |
| class SineGen(torch.nn.Module): | |
| """ Definition of sine generator | |
| SineGen(samp_rate, harmonic_num = 0, | |
| sine_amp = 0.1, noise_std = 0.003, | |
| voiced_threshold = 0, | |
| flag_for_pulse=False) | |
| samp_rate: sampling rate in Hz | |
| harmonic_num: number of harmonic overtones (default 0) | |
| sine_amp: amplitude of sine-wavefrom (default 0.1) | |
| noise_std: std of Gaussian noise (default 0.003) | |
| voiced_thoreshold: F0 threshold for U/V classification (default 0) | |
| flag_for_pulse: this SinGen is used inside PulseGen (default False) | |
| Note: when flag_for_pulse is True, the first time step of a voiced | |
| segment is always sin(np.pi) or cos(0) | |
| """ | |
| def __init__(self, samp_rate, harmonic_num=0, | |
| sine_amp=0.1, noise_std=0.003, | |
| voiced_threshold=0, | |
| flag_for_pulse=False): | |
| super(SineGen, self).__init__() | |
| self.sine_amp = sine_amp | |
| self.noise_std = noise_std | |
| self.harmonic_num = harmonic_num | |
| self.dim = self.harmonic_num + 1 | |
| self.sampling_rate = samp_rate | |
| self.voiced_threshold = voiced_threshold | |
| self.flag_for_pulse = flag_for_pulse | |
| def _f02uv(self, f0): | |
| # generate uv signal | |
| uv = torch.ones_like(f0) | |
| uv = uv * (f0 > self.voiced_threshold) | |
| return uv | |
| def _f02sine(self, f0_values): | |
| """ f0_values: (batchsize, length, dim) | |
| where dim indicates fundamental tone and overtones | |
| """ | |
| # convert to F0 in rad. The interger part n can be ignored | |
| # because 2 * np.pi * n doesn't affect phase | |
| rad_values = (f0_values / self.sampling_rate) % 1 | |
| # initial phase noise (no noise for fundamental component) | |
| rand_ini = torch.rand(f0_values.shape[0], f0_values.shape[2], \ | |
| device=f0_values.device) | |
| rand_ini[:, 0] = 0 | |
| rad_values[:, 0, :] = rad_values[:, 0, :] + rand_ini | |
| # instantanouse phase sine[t] = sin(2*pi \sum_i=1 ^{t} rad) | |
| if not self.flag_for_pulse: | |
| # for normal case | |
| # To prevent torch.cumsum numerical overflow, | |
| # it is necessary to add -1 whenever \sum_k=1^n rad_value_k > 1. | |
| # Buffer tmp_over_one_idx indicates the time step to add -1. | |
| # This will not change F0 of sine because (x-1) * 2*pi = x * 2*pi | |
| tmp_over_one = torch.cumsum(rad_values, 1) % 1 | |
| tmp_over_one_idx = (tmp_over_one[:, 1:, :] - | |
| tmp_over_one[:, :-1, :]) < 0 | |
| cumsum_shift = torch.zeros_like(rad_values) | |
| cumsum_shift[:, 1:, :] = tmp_over_one_idx * -1.0 | |
| sines = torch.sin(torch.cumsum(rad_values + cumsum_shift, dim=1) | |
| * 2 * np.pi) | |
| else: | |
| # If necessary, make sure that the first time step of every | |
| # voiced segments is sin(pi) or cos(0) | |
| # This is used for pulse-train generation | |
| # identify the last time step in unvoiced segments | |
| uv = self._f02uv(f0_values) | |
| uv_1 = torch.roll(uv, shifts=-1, dims=1) | |
| uv_1[:, -1, :] = 1 | |
| u_loc = (uv < 1) * (uv_1 > 0) | |
| # get the instantanouse phase | |
| tmp_cumsum = torch.cumsum(rad_values, dim=1) | |
| # different batch needs to be processed differently | |
| for idx in range(f0_values.shape[0]): | |
| temp_sum = tmp_cumsum[idx, u_loc[idx, :, 0], :] | |
| temp_sum[1:, :] = temp_sum[1:, :] - temp_sum[0:-1, :] | |
| # stores the accumulation of i.phase within | |
| # each voiced segments | |
| tmp_cumsum[idx, :, :] = 0 | |
| tmp_cumsum[idx, u_loc[idx, :, 0], :] = temp_sum | |
| # rad_values - tmp_cumsum: remove the accumulation of i.phase | |
| # within the previous voiced segment. | |
| i_phase = torch.cumsum(rad_values - tmp_cumsum, dim=1) | |
| # get the sines | |
| sines = torch.cos(i_phase * 2 * np.pi) | |
| return sines | |
| def forward(self, f0): | |
| """ sine_tensor, uv = forward(f0) | |
| input F0: tensor(batchsize=1, length, dim=1) | |
| f0 for unvoiced steps should be 0 | |
| output sine_tensor: tensor(batchsize=1, length, dim) | |
| output uv: tensor(batchsize=1, length, 1) | |
| """ | |
| with torch.no_grad(): | |
| f0_buf = torch.zeros(f0.shape[0], f0.shape[1], self.dim, | |
| device=f0.device) | |
| # fundamental component | |
| f0_buf[:, :, 0] = f0[:, :, 0] | |
| for idx in np.arange(self.harmonic_num): | |
| # idx + 2: the (idx+1)-th overtone, (idx+2)-th harmonic | |
| f0_buf[:, :, idx + 1] = f0_buf[:, :, 0] * (idx + 2) | |
| # generate sine waveforms | |
| sine_waves = self._f02sine(f0_buf) * self.sine_amp | |
| # generate uv signal | |
| # uv = torch.ones(f0.shape) | |
| # uv = uv * (f0 > self.voiced_threshold) | |
| uv = self._f02uv(f0) | |
| # noise: for unvoiced should be similar to sine_amp | |
| # std = self.sine_amp/3 -> max value ~ self.sine_amp | |
| # . for voiced regions is self.noise_std | |
| noise_amp = uv * self.noise_std + (1 - uv) * self.sine_amp / 3 | |
| noise = noise_amp * torch.randn_like(sine_waves) | |
| # first: set the unvoiced part to 0 by uv | |
| # then: additive noise | |
| sine_waves = sine_waves * uv + noise | |
| return sine_waves, uv, noise | |
| class PulseGen(torch.nn.Module): | |
| """ Definition of Pulse train generator | |
| There are many ways to implement pulse generator. | |
| Here, PulseGen is based on SinGen. For a perfect | |
| """ | |
| def __init__(self, samp_rate, pulse_amp = 0.1, | |
| noise_std = 0.003, voiced_threshold = 0): | |
| super(PulseGen, self).__init__() | |
| self.pulse_amp = pulse_amp | |
| self.sampling_rate = samp_rate | |
| self.voiced_threshold = voiced_threshold | |
| self.noise_std = noise_std | |
| self.l_sinegen = SineGen(self.sampling_rate, harmonic_num=0, \ | |
| sine_amp=self.pulse_amp, noise_std=0, \ | |
| voiced_threshold=self.voiced_threshold, \ | |
| flag_for_pulse=True) | |
| def forward(self, f0): | |
| """ Pulse train generator | |
| pulse_train, uv = forward(f0) | |
| input F0: tensor(batchsize=1, length, dim=1) | |
| f0 for unvoiced steps should be 0 | |
| output pulse_train: tensor(batchsize=1, length, dim) | |
| output uv: tensor(batchsize=1, length, 1) | |
| Note: self.l_sine doesn't make sure that the initial phase of | |
| a voiced segment is np.pi, the first pulse in a voiced segment | |
| may not be at the first time step within a voiced segment | |
| """ | |
| with torch.no_grad(): | |
| sine_wav, uv, noise = self.l_sinegen(f0) | |
| # sine without additive noise | |
| pure_sine = sine_wav - noise | |
| # step t corresponds to a pulse if | |
| # sine[t] > sine[t+1] & sine[t] > sine[t-1] | |
| # & sine[t-1], sine[t+1], and sine[t] are voiced | |
| # or | |
| # sine[t] is voiced, sine[t-1] is unvoiced | |
| # we use torch.roll to simulate sine[t+1] and sine[t-1] | |
| sine_1 = torch.roll(pure_sine, shifts=1, dims=1) | |
| uv_1 = torch.roll(uv, shifts=1, dims=1) | |
| uv_1[:, 0, :] = 0 | |
| sine_2 = torch.roll(pure_sine, shifts=-1, dims=1) | |
| uv_2 = torch.roll(uv, shifts=-1, dims=1) | |
| uv_2[:, -1, :] = 0 | |
| loc = (pure_sine > sine_1) * (pure_sine > sine_2) \ | |
| * (uv_1 > 0) * (uv_2 > 0) * (uv > 0) \ | |
| + (uv_1 < 1) * (uv > 0) | |
| # pulse train without noise | |
| pulse_train = pure_sine * loc | |
| # additive noise to pulse train | |
| # note that noise from sinegen is zero in voiced regions | |
| pulse_noise = torch.randn_like(pure_sine) * self.noise_std | |
| # with additive noise on pulse, and unvoiced regions | |
| pulse_train += pulse_noise * loc + pulse_noise * (1 - uv) | |
| return pulse_train, sine_wav, uv, pulse_noise | |
| class SignalsConv1d(torch.nn.Module): | |
| """ Filtering input signal with time invariant filter | |
| Note: FIRFilter conducted filtering given fixed FIR weight | |
| SignalsConv1d convolves two signals | |
| Note: this is based on torch.nn.functional.conv1d | |
| """ | |
| def __init__(self): | |
| super(SignalsConv1d, self).__init__() | |
| def forward(self, signal, system_ir): | |
| """ output = forward(signal, system_ir) | |
| signal: (batchsize, length1, dim) | |
| system_ir: (length2, dim) | |
| output: (batchsize, length1, dim) | |
| """ | |
| if signal.shape[-1] != system_ir.shape[-1]: | |
| print("Error: SignalsConv1d expects shape:") | |
| print("signal (batchsize, length1, dim)") | |
| print("system_id (batchsize, length2, dim)") | |
| print("But received signal: {:s}".format(str(signal.shape))) | |
| print(" system_ir: {:s}".format(str(system_ir.shape))) | |
| sys.exit(1) | |
| padding_length = system_ir.shape[0] - 1 | |
| groups = signal.shape[-1] | |
| # pad signal on the left | |
| signal_pad = torch_nn_func.pad(signal.permute(0, 2, 1), \ | |
| (padding_length, 0)) | |
| # prepare system impulse response as (dim, 1, length2) | |
| # also flip the impulse response | |
| ir = torch.flip(system_ir.unsqueeze(1).permute(2, 1, 0), \ | |
| dims=[2]) | |
| # convolute | |
| output = torch_nn_func.conv1d(signal_pad, ir, groups=groups) | |
| return output.permute(0, 2, 1) | |
| class CyclicNoiseGen_v1(torch.nn.Module): | |
| """ CyclicnoiseGen_v1 | |
| Cyclic noise with a single parameter of beta. | |
| Pytorch v1 implementation assumes f_t is also fixed | |
| """ | |
| def __init__(self, samp_rate, | |
| noise_std=0.003, voiced_threshold=0): | |
| super(CyclicNoiseGen_v1, self).__init__() | |
| self.samp_rate = samp_rate | |
| self.noise_std = noise_std | |
| self.voiced_threshold = voiced_threshold | |
| self.l_pulse = PulseGen(samp_rate, pulse_amp=1.0, | |
| noise_std=noise_std, | |
| voiced_threshold=voiced_threshold) | |
| self.l_conv = SignalsConv1d() | |
| def noise_decay(self, beta, f0mean): | |
| """ decayed_noise = noise_decay(beta, f0mean) | |
| decayed_noise = n[t]exp(-t * f_mean / beta / samp_rate) | |
| beta: (dim=1) or (batchsize=1, 1, dim=1) | |
| f0mean (batchsize=1, 1, dim=1) | |
| decayed_noise (batchsize=1, length, dim=1) | |
| """ | |
| with torch.no_grad(): | |
| # exp(-1.0 n / T) < 0.01 => n > -log(0.01)*T = 4.60*T | |
| # truncate the noise when decayed by -40 dB | |
| length = 4.6 * self.samp_rate / f0mean | |
| length = length.int() | |
| time_idx = torch.arange(0, length, device=beta.device) | |
| time_idx = time_idx.unsqueeze(0).unsqueeze(2) | |
| time_idx = time_idx.repeat(beta.shape[0], 1, beta.shape[2]) | |
| noise = torch.randn(time_idx.shape, device=beta.device) | |
| # due to Pytorch implementation, use f0_mean as the f0 factor | |
| decay = torch.exp(-time_idx * f0mean / beta / self.samp_rate) | |
| return noise * self.noise_std * decay | |
| def forward(self, f0s, beta): | |
| """ Producde cyclic-noise | |
| """ | |
| # pulse train | |
| pulse_train, sine_wav, uv, noise = self.l_pulse(f0s) | |
| pure_pulse = pulse_train - noise | |
| # decayed_noise (length, dim=1) | |
| if (uv < 1).all(): | |
| # all unvoiced | |
| cyc_noise = torch.zeros_like(sine_wav) | |
| else: | |
| f0mean = f0s[uv > 0].mean() | |
| decayed_noise = self.noise_decay(beta, f0mean)[0, :, :] | |
| # convolute | |
| cyc_noise = self.l_conv(pure_pulse, decayed_noise) | |
| # add noise in invoiced segments | |
| cyc_noise = cyc_noise + noise * (1.0 - uv) | |
| return cyc_noise, pulse_train, sine_wav, uv, noise | |
| class SineGen(torch.nn.Module): | |
| """ Definition of sine generator | |
| SineGen(samp_rate, harmonic_num = 0, | |
| sine_amp = 0.1, noise_std = 0.003, | |
| voiced_threshold = 0, | |
| flag_for_pulse=False) | |
| samp_rate: sampling rate in Hz | |
| harmonic_num: number of harmonic overtones (default 0) | |
| sine_amp: amplitude of sine-wavefrom (default 0.1) | |
| noise_std: std of Gaussian noise (default 0.003) | |
| voiced_thoreshold: F0 threshold for U/V classification (default 0) | |
| flag_for_pulse: this SinGen is used inside PulseGen (default False) | |
| Note: when flag_for_pulse is True, the first time step of a voiced | |
| segment is always sin(np.pi) or cos(0) | |
| """ | |
| def __init__(self, samp_rate, harmonic_num=0, | |
| sine_amp=0.1, noise_std=0.003, | |
| voiced_threshold=0, | |
| flag_for_pulse=False): | |
| super(SineGen, self).__init__() | |
| self.sine_amp = sine_amp | |
| self.noise_std = noise_std | |
| self.harmonic_num = harmonic_num | |
| self.dim = self.harmonic_num + 1 | |
| self.sampling_rate = samp_rate | |
| self.voiced_threshold = voiced_threshold | |
| self.flag_for_pulse = flag_for_pulse | |
| def _f02uv(self, f0): | |
| # generate uv signal | |
| uv = torch.ones_like(f0) | |
| uv = uv * (f0 > self.voiced_threshold) | |
| return uv | |
| def _f02sine(self, f0_values): | |
| """ f0_values: (batchsize, length, dim) | |
| where dim indicates fundamental tone and overtones | |
| """ | |
| # convert to F0 in rad. The interger part n can be ignored | |
| # because 2 * np.pi * n doesn't affect phase | |
| rad_values = (f0_values / self.sampling_rate) % 1 | |
| # initial phase noise (no noise for fundamental component) | |
| rand_ini = torch.rand(f0_values.shape[0], f0_values.shape[2], \ | |
| device=f0_values.device) | |
| rand_ini[:, 0] = 0 | |
| rad_values[:, 0, :] = rad_values[:, 0, :] + rand_ini | |
| # instantanouse phase sine[t] = sin(2*pi \sum_i=1 ^{t} rad) | |
| if not self.flag_for_pulse: | |
| # for normal case | |
| # To prevent torch.cumsum numerical overflow, | |
| # it is necessary to add -1 whenever \sum_k=1^n rad_value_k > 1. | |
| # Buffer tmp_over_one_idx indicates the time step to add -1. | |
| # This will not change F0 of sine because (x-1) * 2*pi = x * 2*pi | |
| tmp_over_one = torch.cumsum(rad_values, 1) % 1 | |
| tmp_over_one_idx = (tmp_over_one[:, 1:, :] - | |
| tmp_over_one[:, :-1, :]) < 0 | |
| cumsum_shift = torch.zeros_like(rad_values) | |
| cumsum_shift[:, 1:, :] = tmp_over_one_idx * -1.0 | |
| sines = torch.sin(torch.cumsum(rad_values + cumsum_shift, dim=1) | |
| * 2 * np.pi) | |
| else: | |
| # If necessary, make sure that the first time step of every | |
| # voiced segments is sin(pi) or cos(0) | |
| # This is used for pulse-train generation | |
| # identify the last time step in unvoiced segments | |
| uv = self._f02uv(f0_values) | |
| uv_1 = torch.roll(uv, shifts=-1, dims=1) | |
| uv_1[:, -1, :] = 1 | |
| u_loc = (uv < 1) * (uv_1 > 0) | |
| # get the instantanouse phase | |
| tmp_cumsum = torch.cumsum(rad_values, dim=1) | |
| # different batch needs to be processed differently | |
| for idx in range(f0_values.shape[0]): | |
| temp_sum = tmp_cumsum[idx, u_loc[idx, :, 0], :] | |
| temp_sum[1:, :] = temp_sum[1:, :] - temp_sum[0:-1, :] | |
| # stores the accumulation of i.phase within | |
| # each voiced segments | |
| tmp_cumsum[idx, :, :] = 0 | |
| tmp_cumsum[idx, u_loc[idx, :, 0], :] = temp_sum | |
| # rad_values - tmp_cumsum: remove the accumulation of i.phase | |
| # within the previous voiced segment. | |
| i_phase = torch.cumsum(rad_values - tmp_cumsum, dim=1) | |
| # get the sines | |
| sines = torch.cos(i_phase * 2 * np.pi) | |
| return sines | |
| def forward(self, f0): | |
| """ sine_tensor, uv = forward(f0) | |
| input F0: tensor(batchsize=1, length, dim=1) | |
| f0 for unvoiced steps should be 0 | |
| output sine_tensor: tensor(batchsize=1, length, dim) | |
| output uv: tensor(batchsize=1, length, 1) | |
| """ | |
| with torch.no_grad(): | |
| f0_buf = torch.zeros(f0.shape[0], f0.shape[1], self.dim, \ | |
| device=f0.device) | |
| # fundamental component | |
| f0_buf[:, :, 0] = f0[:, :, 0] | |
| for idx in np.arange(self.harmonic_num): | |
| # idx + 2: the (idx+1)-th overtone, (idx+2)-th harmonic | |
| f0_buf[:, :, idx + 1] = f0_buf[:, :, 0] * (idx + 2) | |
| # generate sine waveforms | |
| sine_waves = self._f02sine(f0_buf) * self.sine_amp | |
| # generate uv signal | |
| # uv = torch.ones(f0.shape) | |
| # uv = uv * (f0 > self.voiced_threshold) | |
| uv = self._f02uv(f0) | |
| # noise: for unvoiced should be similar to sine_amp | |
| # std = self.sine_amp/3 -> max value ~ self.sine_amp | |
| # . for voiced regions is self.noise_std | |
| noise_amp = uv * self.noise_std + (1 - uv) * self.sine_amp / 3 | |
| noise = noise_amp * torch.randn_like(sine_waves) | |
| # first: set the unvoiced part to 0 by uv | |
| # then: additive noise | |
| sine_waves = sine_waves * uv + noise | |
| return sine_waves, uv, noise | |
| class SourceModuleCycNoise_v1(torch.nn.Module): | |
| """ SourceModuleCycNoise_v1 | |
| SourceModule(sampling_rate, noise_std=0.003, voiced_threshod=0) | |
| sampling_rate: sampling_rate in Hz | |
| noise_std: std of Gaussian noise (default: 0.003) | |
| voiced_threshold: threshold to set U/V given F0 (default: 0) | |
| cyc, noise, uv = SourceModuleCycNoise_v1(F0_upsampled, beta) | |
| F0_upsampled (batchsize, length, 1) | |
| beta (1) | |
| cyc (batchsize, length, 1) | |
| noise (batchsize, length, 1) | |
| uv (batchsize, length, 1) | |
| """ | |
| def __init__(self, sampling_rate, noise_std=0.003, voiced_threshod=0): | |
| super(SourceModuleCycNoise_v1, self).__init__() | |
| self.sampling_rate = sampling_rate | |
| self.noise_std = noise_std | |
| self.l_cyc_gen = CyclicNoiseGen_v1(sampling_rate, noise_std, | |
| voiced_threshod) | |
| def forward(self, f0_upsamped, beta): | |
| """ | |
| cyc, noise, uv = SourceModuleCycNoise_v1(F0, beta) | |
| F0_upsampled (batchsize, length, 1) | |
| beta (1) | |
| cyc (batchsize, length, 1) | |
| noise (batchsize, length, 1) | |
| uv (batchsize, length, 1) | |
| """ | |
| # source for harmonic branch | |
| cyc, pulse, sine, uv, add_noi = self.l_cyc_gen(f0_upsamped, beta) | |
| # source for noise branch, in the same shape as uv | |
| noise = torch.randn_like(uv) * self.noise_std / 3 | |
| return cyc, noise, uv | |
| class SourceModuleHnNSF(torch.nn.Module): | |
| """ SourceModule for hn-nsf | |
| SourceModule(sampling_rate, harmonic_num=0, sine_amp=0.1, | |
| add_noise_std=0.003, voiced_threshod=0) | |
| sampling_rate: sampling_rate in Hz | |
| harmonic_num: number of harmonic above F0 (default: 0) | |
| sine_amp: amplitude of sine source signal (default: 0.1) | |
| add_noise_std: std of additive Gaussian noise (default: 0.003) | |
| note that amplitude of noise in unvoiced is decided | |
| by sine_amp | |
| voiced_threshold: threhold to set U/V given F0 (default: 0) | |
| Sine_source, noise_source = SourceModuleHnNSF(F0_sampled) | |
| F0_sampled (batchsize, length, 1) | |
| Sine_source (batchsize, length, 1) | |
| noise_source (batchsize, length 1) | |
| uv (batchsize, length, 1) | |
| """ | |
| def __init__(self, sampling_rate, harmonic_num=0, sine_amp=0.1, | |
| add_noise_std=0.003, voiced_threshod=0): | |
| super(SourceModuleHnNSF, self).__init__() | |
| self.sine_amp = sine_amp | |
| self.noise_std = add_noise_std | |
| # to produce sine waveforms | |
| self.l_sin_gen = SineGen(sampling_rate, harmonic_num, | |
| sine_amp, add_noise_std, voiced_threshod) | |
| # to merge source harmonics into a single excitation | |
| self.l_linear = torch.nn.Linear(harmonic_num + 1, 1) | |
| self.l_tanh = torch.nn.Tanh() | |
| def forward(self, x): | |
| """ | |
| Sine_source, noise_source = SourceModuleHnNSF(F0_sampled) | |
| F0_sampled (batchsize, length, 1) | |
| Sine_source (batchsize, length, 1) | |
| noise_source (batchsize, length 1) | |
| """ | |
| # source for harmonic branch | |
| sine_wavs, uv, _ = self.l_sin_gen(x) | |
| sine_merge = self.l_tanh(self.l_linear(sine_wavs)) | |
| # source for noise branch, in the same shape as uv | |
| noise = torch.randn_like(uv) * self.sine_amp / 3 | |
| return sine_merge, noise, uv | |
| if __name__ == '__main__': | |
| source = SourceModuleCycNoise_v1(24000) | |
| x = torch.randn(16, 25600, 1) | |