+ for p in group["params"]:
+ state = self.state[p]
+ ...
+
+ you can do:
+
+ with self.batched_params(group["params"]) as batches:
+ for p, state, p_names in batches:
+ ...
+
+
+ Args:
+ group: a parameter group, which is a list of parameters; should be
+ one of self.param_groups.
+ group_params_names: name for each parameter in group,
+ which is List[str].
+ """
+ batches = defaultdict(
+ list
+ ) # `batches` maps from tuple (dtype_as_str,*shape) to list of nn.Parameter
+ batches_names = defaultdict(
+ list
+ ) # `batches` maps from tuple (dtype_as_str,*shape) to list of str
+
+ assert len(param_group) == len(group_params_names)
+ for p, named_p in zip(param_group, group_params_names):
+ key = (str(p.dtype), *p.shape)
+ batches[key].append(p)
+ batches_names[key].append(named_p)
+
+ batches_names_keys = list(batches_names.keys())
+ sorted_idx = sorted(
+ range(len(batches_names)), key=lambda i: batches_names_keys[i])
+ batches_names = [
+ batches_names[batches_names_keys[idx]] for idx in sorted_idx
+ ]
+ batches = [batches[batches_names_keys[idx]] for idx in sorted_idx]
+
+ stacked_params_dict = dict()
+
+ # turn batches into a list, in deterministic order.
+ # tuples will contain tuples of (stacked_param, state, stacked_params_names),
+ # one for each batch in `batches`.
+ tuples = []
+
+ for batch, batch_names in zip(batches, batches_names):
+ p = batch[0]
+ # we arbitrarily store the state in the
+ # state corresponding to the 1st parameter in the
+ # group. class Optimizer will take care of saving/loading state.
+ state = self.state[p]
+ p_stacked = torch.stack(batch)
+ grad = torch.stack([
+ torch.zeros_like(p) if p.grad is None else p.grad for p in batch
+ ])
+ p_stacked.grad = grad
+ stacked_params_dict[key] = p_stacked
+ tuples.append((p_stacked, state, batch_names))
+
+ yield tuples # <-- calling code will do the actual optimization here!
+
+ for ((stacked_params, _state, _names), batch) in zip(tuples, batches):
+ for i, p in enumerate(batch): # batch is list of Parameter
+ p.copy_(stacked_params[i])
+
+
+class ScaledAdam(BatchedOptimizer):
+ """
+ Implements 'Scaled Adam', a variant of Adam where we scale each parameter's update
+ proportional to the norm of that parameter; and also learn the scale of the parameter,
+ in log space, subject to upper and lower limits (as if we had factored each parameter as
+ param = underlying_param * log_scale.exp())
+
+
+ Args:
+ params: The parameters or param_groups to optimize (like other Optimizer subclasses)
+ lr: The learning rate. We will typically use a learning rate schedule that starts
+ at 0.03 and decreases over time, i.e. much higher than other common
+ optimizers.
+ clipping_scale: (e.g. 2.0)
+ A scale for gradient-clipping: if specified, the normalized gradients
+ over the whole model will be clipped to have 2-norm equal to
+ `clipping_scale` times the median 2-norm over the most recent period
+ of `clipping_update_period` minibatches. By "normalized gradients",
+ we mean after multiplying by the rms parameter value for this tensor
+ [for non-scalars]; this is appropriate because our update is scaled
+ by this quantity.
+ betas: beta1,beta2 are momentum constants for regular momentum, and moving sum-sq grad.
+ Must satisfy 0 < beta <= beta2 < 1.
+ scalar_lr_scale: A scaling factor on the learning rate, that we use to update the
+ scale of each parameter tensor and scalar parameters of the mode..
+ If each parameter were decomposed
+ as p * p_scale.exp(), where (p**2).mean().sqrt() == 1.0, scalar_lr_scale
+ would be a the scaling factor on the learning rate of p_scale.
+ eps: A general-purpose epsilon to prevent division by zero
+ param_min_rms: Minimum root-mean-square value of parameter tensor, for purposes of
+ learning the scale on the parameters (we'll constrain the rms of each non-scalar
+ parameter tensor to be >= this value)
+ param_max_rms: Maximum root-mean-square value of parameter tensor, for purposes of
+ learning the scale on the parameters (we'll constrain the rms of each non-scalar
+ parameter tensor to be <= this value)
+ scalar_max: Maximum absolute value for scalar parameters (applicable if your
+ model has any parameters with numel() == 1).
+ size_update_period: The periodicity, in steps, with which we update the size (scale)
+ of the parameter tensor. This is provided to save a little time
+ in the update.
+ clipping_update_period: if clipping_scale is specified, this is the period
+ """
+
+ def __init__(
+ self,
+ params,
+ lr=3e-02,
+ clipping_scale=None,
+ betas=(0.9, 0.98),
+ scalar_lr_scale=0.1,
+ eps=1.0e-08,
+ param_min_rms=1.0e-05,
+ param_max_rms=3.0,
+ scalar_max=10.0,
+ size_update_period=4,
+ clipping_update_period=100,
+ parameters_names=None,
+ show_dominant_parameters=True, ):
+
+ assert parameters_names is not None, (
+ "Please prepare parameters_names,"
+ "which is a List[List[str]]. Each List[str] is for a group"
+ "and each str is for a parameter")
+ defaults = dict(
+ lr=lr,
+ clipping_scale=clipping_scale,
+ betas=betas,
+ scalar_lr_scale=scalar_lr_scale,
+ eps=eps,
+ param_min_rms=param_min_rms,
+ param_max_rms=param_max_rms,
+ scalar_max=scalar_max,
+ size_update_period=size_update_period,
+ clipping_update_period=clipping_update_period, )
+
+ super(ScaledAdam, self).__init__(params, defaults)
+ assert len(self.param_groups) == len(parameters_names)
+ self.parameters_names = parameters_names
+ self.show_dominant_parameters = show_dominant_parameters
+
+ def __setstate__(self, state):
+ super(ScaledAdam, self).__setstate__(state)
+
+ @torch.no_grad()
+ def step(self, closure=None):
+ """Performs a single optimization step.
+
+ Arguments:
+ closure (callable, optional): A closure that reevaluates the model
+ and returns the loss.
+ """
+ loss = None
+ if closure is not None:
+ with torch.enable_grad():
+ loss = closure()
+
+ batch = True
+
+ for group, group_params_names in zip(self.param_groups,
+ self.parameters_names):
+
+ with self.batched_params(group["params"],
+ group_params_names) as batches:
+
+ # batches is list of pairs (stacked_param, state). stacked_param is like
+ # a regular parameter, and will have a .grad, but the 1st dim corresponds to
+ # a stacking dim, it is not a real dim.
+
+ if (len(batches[0][1]) ==
+ 0): # if len(first state) == 0: not yet initialized
+ clipping_scale = 1
+ else:
+ clipping_scale = self._get_clipping_scale(group, batches)
+
+ for p, state, _ in batches:
+ # Perform optimization step.
+ # grad is not going to be None, we handled that when creating the batches.
+ grad = p.grad
+ if grad.is_sparse:
+ raise RuntimeError(
+ "ScaledAdam optimizer does not support sparse gradients"
+ )
+ # State initialization
+ if len(state) == 0:
+ self._init_state(group, p, state)
+
+ self._step_one_batch(group, p, state, clipping_scale)
+
+ return loss
+
+ def _init_state(self, group: dict, p: Tensor, state: dict):
+ """
+ Initializes state dict for parameter 'p'. Assumes that dim 0 of tensor p
+ is actually the batch dimension, corresponding to batched-together
+ parameters of a given shape.
+
+
+ Args:
+ group: Dict to look up configuration values.
+ p: The parameter that we are initializing the state for
+ state: Dict from string to whatever state we are initializing
+ """
+ size_update_period = group["size_update_period"]
+
+ state["step"] = 0
+
+ kwargs = {"device": p.device, "dtype": p.dtype}
+
+ # 'delta' implements conventional momentum. There are
+ # several different kinds of update going on, so rather than
+ # compute "exp_avg" like in Adam, we store and decay a
+ # parameter-change "delta", which combines all forms of
+ # update. this is equivalent to how it's done in Adam,
+ # except for the first few steps.
+ state["delta"] = torch.zeros_like(
+ p, memory_format=torch.preserve_format)
+
+ batch_size = p.shape[0]
+ numel = p.numel() // batch_size
+ numel = p.numel()
+
+ if numel > 1:
+ # "param_rms" just periodically records the scalar root-mean-square value of
+ # the parameter tensor.
+ # it has a shape like (batch_size, 1, 1, 1, 1)
+ param_rms = (
+ (p**2).mean(dim=list(range(1, p.ndim)), keepdim=True).sqrt())
+ state["param_rms"] = param_rms
+
+ state["scale_exp_avg_sq"] = torch.zeros_like(param_rms)
+ state["scale_grads"] = torch.zeros(size_update_period,
+ *param_rms.shape, **kwargs)
+
+ # exp_avg_sq is the weighted sum of scaled gradients. as in Adam.
+ state["exp_avg_sq"] = torch.zeros_like(
+ p, memory_format=torch.preserve_format)
+
+ def _get_clipping_scale(self,
+ group: dict,
+ tuples: List[Tuple[Tensor, dict, List[str]]]
+ ) -> float:
+ """
+ Returns a scalar factor <= 1.0 that dictates gradient clipping, i.e. we will scale the gradients
+ by this amount before applying the rest of the update.
+
+ Args:
+ group: the parameter group, an item in self.param_groups
+ tuples: a list of tuples of (param, state, param_names)
+ where param is a batched set of parameters,
+ with a .grad (1st dim is batch dim)
+ and state is the state-dict where optimization parameters are kept.
+ param_names is a List[str] while each str is name for a parameter
+ in batched set of parameters "param".
+ """
+ assert len(tuples) >= 1
+ clipping_scale = group["clipping_scale"]
+ (first_p, first_state, _) = tuples[0]
+ step = first_state["step"]
+ if clipping_scale is None or step == 0:
+ # no clipping. return early on step == 0 because the other
+ # parameters' state won't have been initialized yet.
+ return 1.0
+ clipping_update_period = group["clipping_update_period"]
+
+ tot_sumsq = torch.tensor(0.0, device=first_p.device)
+ for (p, state, param_names) in tuples:
+ grad = p.grad
+ if grad.is_sparse:
+ raise RuntimeError(
+ "ScaledAdam optimizer does not support sparse gradients")
+ if p.numel() == p.shape[0]: # a batch of scalars
+ tot_sumsq += (grad**2).sum() # sum() to change shape [1] to []
+ else:
+ tot_sumsq += ((grad * state["param_rms"])**2).sum()
+
+ tot_norm = tot_sumsq.sqrt()
+ if "model_norms" not in first_state:
+ first_state["model_norms"] = torch.zeros(
+ clipping_update_period, device=p.device)
+ first_state["model_norms"][step % clipping_update_period] = tot_norm
+
+ if step % clipping_update_period == 0:
+ # Print some stats.
+ # We don't reach here if step == 0 because we would have returned
+ # above.
+ sorted_norms = first_state["model_norms"].sort()[0].to("cpu")
+ quartiles = []
+ for n in range(0, 5):
+ index = min(
+ clipping_update_period - 1,
+ (clipping_update_period // 4) * n, )
+ quartiles.append(sorted_norms[index].item())
+
+ median = quartiles[2]
+ threshold = clipping_scale * median
+ first_state["model_norm_threshold"] = threshold
+ percent_clipped = (first_state["num_clipped"] * 100.0 /
+ clipping_update_period
+ if "num_clipped" in first_state else 0.0)
+ first_state["num_clipped"] = 0
+ quartiles = " ".join(["%.3e" % x for x in quartiles])
+ logging.info(
+ f"Clipping_scale={clipping_scale}, grad-norm quartiles {quartiles}, "
+ f"threshold={threshold:.3e}, percent-clipped={percent_clipped:.1f}"
+ )
+
+ if step < clipping_update_period:
+ return 1.0 # We have not yet estimated a norm to clip to.
+ else:
+ try:
+ model_norm_threshold = first_state["model_norm_threshold"]
+ except KeyError:
+ logging.info(
+ "Warning: model_norm_threshold not in state: possibly "
+ "you changed config when restarting, adding clipping_scale option?"
+ )
+ return 1.0
+ ans = min(1.0, (model_norm_threshold / (tot_norm + 1.0e-20)).item())
+ if ans < 1.0:
+ first_state["num_clipped"] += 1
+ if ans < 0.1:
+ logging.warn(
+ f"Scaling gradients by {ans}, model_norm_threshold={model_norm_threshold}"
+ )
+ if self.show_dominant_parameters:
+ assert p.shape[0] == len(param_names)
+ self._show_gradient_dominating_parameter(tuples, tot_sumsq)
+ return ans
+
+ def _show_gradient_dominating_parameter(
+ self, tuples: List[Tuple[Tensor, dict, List[str]]],
+ tot_sumsq: Tensor):
+ """
+ Show information of parameter wihch dominanting tot_sumsq.
+
+ Args:
+ tuples: a list of tuples of (param, state, param_names)
+ where param is a batched set of parameters,
+ with a .grad (1st dim is batch dim)
+ and state is the state-dict where optimization parameters are kept.
+ param_names is a List[str] while each str is name for a parameter
+ in batched set of parameters "param".
+ tot_sumsq: sumsq of all parameters. Though it's could be calculated
+ from tuples, we still pass it to save some time.
+ """
+ all_sumsq_orig = {}
+ for (p, state, batch_param_names) in tuples:
+ # p is a stacked batch parameters.
+ batch_grad = p.grad
+ if p.numel() == p.shape[0]: # a batch of scalars
+ batch_sumsq_orig = batch_grad**2
+ # Dummpy values used by following `zip` statement.
+ batch_rms_orig = torch.ones(p.shape[0])
+ else:
+ batch_rms_orig = state["param_rms"]
+ batch_sumsq_orig = ((batch_grad * batch_rms_orig)**2).sum(
+ dim=list(range(1, batch_grad.ndim)))
+
+ for name, sumsq_orig, rms, grad in zip(batch_param_names,
+ batch_sumsq_orig,
+ batch_rms_orig, batch_grad):
+
+ proportion_orig = sumsq_orig / tot_sumsq
+ all_sumsq_orig[name] = (proportion_orig, sumsq_orig, rms, grad)
+
+ assert torch.isclose(
+ sum([value[0] for value in all_sumsq_orig.values()]).cpu(),
+ torch.tensor(1.0), )
+ sorted_by_proportion = {
+ k: v
+ for k, v in sorted(
+ all_sumsq_orig.items(),
+ key=lambda item: item[1][0],
+ reverse=True, )
+ }
+ dominant_param_name = next(iter(sorted_by_proportion))
+ (dominant_proportion, dominant_sumsq, dominant_rms,
+ dominant_grad, ) = sorted_by_proportion[dominant_param_name]
+ logging.info(f"Parameter Dominanting tot_sumsq {dominant_param_name}"
+ f" with proportion {dominant_proportion:.2f},"
+ f" where dominant_sumsq=(grad_sumsq*orig_rms_sq)"
+ f"={dominant_sumsq:.3e},"
+ f" grad_sumsq = {(dominant_grad**2).sum():.3e},"
+ f" orig_rms_sq={(dominant_rms**2).item():.3e}")
+
+ def _step_one_batch(self,
+ group: dict,
+ p: Tensor,
+ state: dict,
+ clipping_scale: float):
+ """
+ Do the step for one parameter, which is actually going to be a batch of
+ `real` parameters, with dim 0 as the batch dim.
+ Args:
+ group: dict to look up configuration values
+ p: parameter to update (actually multiple parameters stacked together
+ as a batch)
+ state: state-dict for p, to look up the optimizer state
+ """
+ lr = group["lr"]
+ size_update_period = group["size_update_period"]
+ beta1 = group["betas"][0]
+
+ grad = p.grad
+ if clipping_scale != 1.0:
+ grad = grad * clipping_scale
+ step = state["step"]
+ delta = state["delta"]
+
+ delta.mul_(beta1)
+ batch_size = p.shape[0]
+ numel = p.numel() // batch_size
+ if numel > 1:
+ # Update the size/scale of p, and set param_rms
+ scale_grads = state["scale_grads"]
+ scale_grads[step % size_update_period] = (p * grad).sum(
+ dim=list(range(1, p.ndim)), keepdim=True)
+ if step % size_update_period == size_update_period - 1:
+ param_rms = state["param_rms"] # shape: (batch_size, 1, 1, ..)
+ param_rms.copy_((p**2)
+ .mean(dim=list(range(1, p.ndim)), keepdim=True)
+ .sqrt())
+ if step > 0:
+ # self._size_update() learns the overall scale on the
+ # parameter, by shrinking or expanding it.
+ self._size_update(group, scale_grads, p, state)
+
+ if numel == 1:
+ # For parameters with 1 element we just use regular Adam.
+ # Updates delta.
+ self._step_scalar(group, p, state)
+ else:
+ self._step(group, p, state)
+
+ state["step"] = step + 1
+
+ def _size_update(self,
+ group: dict,
+ scale_grads: Tensor,
+ p: Tensor,
+ state: dict) -> None:
+ """
+ Called only where p.numel() > 1, this updates the scale of the parameter.
+ If we imagine: p = underlying_param * scale.exp(), and we are doing
+ gradient descent on underlying param and on scale, this function does the update
+ on `scale`.
+
+ Args:
+ group: dict to look up configuration values
+ scale_grads: a tensor of shape (size_update_period, batch_size, 1, 1,...) containing
+ grads w.r.t. the scales.
+ p: The parameter to update
+ state: The state-dict of p
+ """
+
+ param_rms = state["param_rms"]
+ beta1, beta2 = group["betas"]
+ size_lr = group["lr"] * group["scalar_lr_scale"]
+ param_min_rms = group["param_min_rms"]
+ param_max_rms = group["param_max_rms"]
+ eps = group["eps"]
+ step = state["step"]
+ batch_size = p.shape[0]
+
+ size_update_period = scale_grads.shape[0]
+ # correct beta2 for the size update period: we will have
+ # faster decay at this level.
+ beta2_corr = beta2**size_update_period
+
+ scale_exp_avg_sq = state[
+ "scale_exp_avg_sq"] # shape: (batch_size, 1, 1, ..)
+ scale_exp_avg_sq.mul_(beta2_corr).add_(
+ (scale_grads**2).mean(dim=0), # mean over dim `size_update_period`
+ alpha=1 - beta2_corr, ) # shape is (batch_size, 1, 1, ...)
+
+ # The 1st time we reach here is when size_step == 1.
+ size_step = (step + 1) // size_update_period
+ bias_correction2 = 1 - beta2_corr**size_step
+ # we don't bother with bias_correction1; this will help prevent divergence
+ # at the start of training.
+
+ denom = scale_exp_avg_sq.sqrt() + eps
+
+ scale_step = (-size_lr * (bias_correction2**0.5) *
+ scale_grads.sum(dim=0) / denom)
+
+ is_too_small = param_rms < param_min_rms
+ is_too_large = param_rms > param_max_rms
+
+ # when the param gets too small, just don't shrink it any further.
+ scale_step.masked_fill_(is_too_small, 0.0)
+ # when it gets too large, stop it from getting any larger.
+ scale_step.masked_fill_(is_too_large, -size_lr * size_update_period)
+ delta = state["delta"]
+ # the factor of (1-beta1) relates to momentum.
+ delta.add_(p * scale_step, alpha=(1 - beta1))
+
+ def _step(self, group: dict, p: Tensor, state: dict):
+ """
+ This function does the core update of self.step(), in the case where the members of
+ the batch have more than 1 element.
+
+ Args:
+ group: A dict which will be used to look up configuration values
+ p: The parameter to be updated
+ grad: The grad of p
+ state: The state-dict corresponding to parameter p
+
+ This function modifies p.
+ """
+ grad = p.grad
+ lr = group["lr"]
+ beta1, beta2 = group["betas"]
+ eps = group["eps"]
+ param_min_rms = group["param_min_rms"]
+ step = state["step"]
+
+ exp_avg_sq = state["exp_avg_sq"]
+ exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=(1 - beta2))
+
+ this_step = state["step"] - (state["zero_step"]
+ if "zero_step" in state else 0)
+ bias_correction2 = 1 - beta2**(this_step + 1)
+ if bias_correction2 < 0.99:
+ # note: not in-place.
+ exp_avg_sq = exp_avg_sq * (1.0 / bias_correction2)
+
+ denom = exp_avg_sq.sqrt()
+ denom += eps
+ grad = grad / denom
+
+ alpha = -lr * (1 - beta1) * state["param_rms"].clamp(min=param_min_rms)
+
+ delta = state["delta"]
+ delta.add_(grad * alpha)
+ p.add_(delta)
+
+ def _step_scalar(self, group: dict, p: Tensor, state: dict):
+ """
+ A simplified form of the core update for scalar tensors, where we cannot get a good
+ estimate of the parameter rms.
+ """
+ beta1, beta2 = group["betas"]
+ scalar_max = group["scalar_max"]
+ eps = group["eps"]
+ lr = group["lr"] * group["scalar_lr_scale"]
+ grad = p.grad
+
+ exp_avg_sq = state["exp_avg_sq"] # shape: (batch_size,)
+ exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)
+
+ # bias_correction2 is like in Adam. Don't bother with bias_correction1;
+ # slower update at the start will help stability anyway.
+ bias_correction2 = 1 - beta2**(state["step"] + 1)
+ denom = (exp_avg_sq / bias_correction2).sqrt() + eps
+
+ delta = state["delta"]
+ delta.add_(grad / denom, alpha=-lr * (1 - beta1))
+ p.clamp_(min=-scalar_max, max=scalar_max)
+ p.add_(delta)
diff --git a/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/AR/modules/patched_mha_with_cache.py b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/AR/modules/patched_mha_with_cache.py
new file mode 100644
index 0000000000000000000000000000000000000000..77378c0929d51f347d51ebcfb489ab341672ae65
--- /dev/null
+++ b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/AR/modules/patched_mha_with_cache.py
@@ -0,0 +1,388 @@
+from torch.nn.functional import *
+from torch.nn.functional import _mha_shape_check,_canonical_mask,_none_or_dtype,_in_projection_packed
+# import torch
+# Tensor = torch.Tensor
+# from typing import Callable, List, Optional, Tuple, Union
+
+def multi_head_attention_forward_patched(
+ query: Tensor,
+ key: Tensor,
+ value: Tensor,
+ embed_dim_to_check: int,
+ num_heads: int,
+ in_proj_weight: Optional[Tensor],
+ in_proj_bias: Optional[Tensor],
+ bias_k: Optional[Tensor],
+ bias_v: Optional[Tensor],
+ add_zero_attn: bool,
+ dropout_p: float,
+ out_proj_weight: Tensor,
+ out_proj_bias: Optional[Tensor],
+ training: bool = True,
+ key_padding_mask: Optional[Tensor] = None,
+ need_weights: bool = True,
+ attn_mask: Optional[Tensor] = None,
+ use_separate_proj_weight: bool = False,
+ q_proj_weight: Optional[Tensor] = None,
+ k_proj_weight: Optional[Tensor] = None,
+ v_proj_weight: Optional[Tensor] = None,
+ static_k: Optional[Tensor] = None,
+ static_v: Optional[Tensor] = None,
+ average_attn_weights: bool = True,
+ is_causal: bool = False,cache=None
+) -> Tuple[Tensor, Optional[Tensor]]:
+ r"""
+ Args:
+ query, key, value: map a query and a set of key-value pairs to an output.
+ See "Attention Is All You Need" for more details.
+ embed_dim_to_check: total dimension of the model.
+ num_heads: parallel attention heads.
+ in_proj_weight, in_proj_bias: input projection weight and bias.
+ bias_k, bias_v: bias of the key and value sequences to be added at dim=0.
+ add_zero_attn: add a new batch of zeros to the key and
+ value sequences at dim=1.
+ dropout_p: probability of an element to be zeroed.
+ out_proj_weight, out_proj_bias: the output projection weight and bias.
+ training: apply dropout if is ``True``.
+ key_padding_mask: if provided, specified padding elements in the key will
+ be ignored by the attention. This is an binary mask. When the value is True,
+ the corresponding value on the attention layer will be filled with -inf.
+ need_weights: output attn_output_weights.
+ Default: `True`
+ Note: `needs_weight` defaults to `True`, but should be set to `False`
+ For best performance when attention weights are not nedeeded.
+ *Setting needs_weights to `True`
+ leads to a significant performance degradation.*
+ attn_mask: 2D or 3D mask that prevents attention to certain positions. A 2D mask will be broadcasted for all
+ the batches while a 3D mask allows to specify a different mask for the entries of each batch.
+ is_causal: If specified, applies a causal mask as attention mask, and ignores
+ attn_mask for computing scaled dot product attention.
+ Default: ``False``.
+ .. warning::
+ is_causal is provides a hint that the attn_mask is the
+ causal mask.Providing incorrect hints can result in
+ incorrect execution, including forward and backward
+ compatibility.
+ use_separate_proj_weight: the function accept the proj. weights for query, key,
+ and value in different forms. If false, in_proj_weight will be used, which is
+ a combination of q_proj_weight, k_proj_weight, v_proj_weight.
+ q_proj_weight, k_proj_weight, v_proj_weight, in_proj_bias: input projection weight and bias.
+ static_k, static_v: static key and value used for attention operators.
+ average_attn_weights: If true, indicates that the returned ``attn_weights`` should be averaged across heads.
+ Otherwise, ``attn_weights`` are provided separately per head. Note that this flag only has an effect
+ when ``need_weights=True.``. Default: True
+
+
+ Shape:
+ Inputs:
+ - query: :math:`(L, E)` or :math:`(L, N, E)` where L is the target sequence length, N is the batch size, E is
+ the embedding dimension.
+ - key: :math:`(S, E)` or :math:`(S, N, E)`, where S is the source sequence length, N is the batch size, E is
+ the embedding dimension.
+ - value: :math:`(S, E)` or :math:`(S, N, E)` where S is the source sequence length, N is the batch size, E is
+ the embedding dimension.
+ - key_padding_mask: :math:`(S)` or :math:`(N, S)` where N is the batch size, S is the source sequence length.
+ If a FloatTensor is provided, it will be directly added to the value.
+ If a BoolTensor is provided, the positions with the
+ value of ``True`` will be ignored while the position with the value of ``False`` will be unchanged.
+ - attn_mask: 2D mask :math:`(L, S)` where L is the target sequence length, S is the source sequence length.
+ 3D mask :math:`(N*num_heads, L, S)` where N is the batch size, L is the target sequence length,
+ S is the source sequence length. attn_mask ensures that position i is allowed to attend the unmasked
+ positions. If a BoolTensor is provided, positions with ``True``
+ are not allowed to attend while ``False`` values will be unchanged. If a FloatTensor
+ is provided, it will be added to the attention weight.
+ - static_k: :math:`(N*num_heads, S, E/num_heads)`, where S is the source sequence length,
+ N is the batch size, E is the embedding dimension. E/num_heads is the head dimension.
+ - static_v: :math:`(N*num_heads, S, E/num_heads)`, where S is the source sequence length,
+ N is the batch size, E is the embedding dimension. E/num_heads is the head dimension.
+
+ Outputs:
+ - attn_output: :math:`(L, E)` or :math:`(L, N, E)` where L is the target sequence length, N is the batch size,
+ E is the embedding dimension.
+ - attn_output_weights: Only returned when ``need_weights=True``. If ``average_attn_weights=True``, returns
+ attention weights averaged across heads of shape :math:`(L, S)` when input is unbatched or
+ :math:`(N, L, S)`, where :math:`N` is the batch size, :math:`L` is the target sequence length, and
+ :math:`S` is the source sequence length. If ``average_attn_weights=False``, returns attention weights per
+ head of shape :math:`(num_heads, L, S)` when input is unbatched or :math:`(N, num_heads, L, S)`.
+ """
+ tens_ops = (query, key, value, in_proj_weight, in_proj_bias, bias_k, bias_v, out_proj_weight, out_proj_bias)
+ if has_torch_function(tens_ops):
+ return handle_torch_function(
+ multi_head_attention_forward,
+ tens_ops,
+ query,
+ key,
+ value,
+ embed_dim_to_check,
+ num_heads,
+ in_proj_weight,
+ in_proj_bias,
+ bias_k,
+ bias_v,
+ add_zero_attn,
+ dropout_p,
+ out_proj_weight,
+ out_proj_bias,
+ training=training,
+ key_padding_mask=key_padding_mask,
+ need_weights=need_weights,
+ attn_mask=attn_mask,
+ is_causal=is_causal,
+ use_separate_proj_weight=use_separate_proj_weight,
+ q_proj_weight=q_proj_weight,
+ k_proj_weight=k_proj_weight,
+ v_proj_weight=v_proj_weight,
+ static_k=static_k,
+ static_v=static_v,
+ average_attn_weights=average_attn_weights,cache=cache
+ )
+
+ is_batched = _mha_shape_check(query, key, value, key_padding_mask, attn_mask, num_heads)
+
+ # For unbatched input, we unsqueeze at the expected batch-dim to pretend that the input
+ # is batched, run the computation and before returning squeeze the
+ # batch dimension so that the output doesn't carry this temporary batch dimension.
+ if not is_batched:
+ # unsqueeze if the input is unbatched
+ query = query.unsqueeze(1)
+ key = key.unsqueeze(1)
+ value = value.unsqueeze(1)
+ if key_padding_mask is not None:
+ key_padding_mask = key_padding_mask.unsqueeze(0)
+
+ # set up shape vars
+ tgt_len, bsz, embed_dim = query.shape
+ src_len, _, _ = key.shape
+
+ key_padding_mask = _canonical_mask(
+ mask=key_padding_mask,
+ mask_name="key_padding_mask",
+ other_type=_none_or_dtype(attn_mask),
+ other_name="attn_mask",
+ target_type=query.dtype
+ )
+
+ if is_causal and attn_mask is None:
+ raise RuntimeError(
+ "Need attn_mask if specifying the is_causal hint. "
+ "You may use the Transformer module method "
+ "`generate_square_subsequent_mask` to create this mask."
+ )
+
+ if is_causal and key_padding_mask is None and not need_weights:
+ # when we have a kpm or need weights, we need attn_mask
+ # Otherwise, we use the is_causal hint go as is_causal
+ # indicator to SDPA.
+ attn_mask = None
+ else:
+ attn_mask = _canonical_mask(
+ mask=attn_mask,
+ mask_name="attn_mask",
+ other_type=None,
+ other_name="",
+ target_type=query.dtype,
+ check_other=False,
+ )
+
+
+ if key_padding_mask is not None:
+ # We have the attn_mask, and use that to merge kpm into it.
+ # Turn off use of is_causal hint, as the merged mask is no
+ # longer causal.
+ is_causal = False
+
+ assert embed_dim == embed_dim_to_check, \
+ f"was expecting embedding dimension of {embed_dim_to_check}, but got {embed_dim}"
+ if isinstance(embed_dim, torch.Tensor):
+ # embed_dim can be a tensor when JIT tracing
+ head_dim = embed_dim.div(num_heads, rounding_mode='trunc')
+ else:
+ head_dim = embed_dim // num_heads
+ assert head_dim * num_heads == embed_dim, f"embed_dim {embed_dim} not divisible by num_heads {num_heads}"
+ if use_separate_proj_weight:
+ # allow MHA to have different embedding dimensions when separate projection weights are used
+ assert key.shape[:2] == value.shape[:2], \
+ f"key's sequence and batch dims {key.shape[:2]} do not match value's {value.shape[:2]}"
+ else:
+ assert key.shape == value.shape, f"key shape {key.shape} does not match value shape {value.shape}"
+
+ #
+ # compute in-projection
+ #
+ if not use_separate_proj_weight:
+ assert in_proj_weight is not None, "use_separate_proj_weight is False but in_proj_weight is None"
+ q, k, v = _in_projection_packed(query, key, value, in_proj_weight, in_proj_bias)
+ else:
+ assert q_proj_weight is not None, "use_separate_proj_weight is True but q_proj_weight is None"
+ assert k_proj_weight is not None, "use_separate_proj_weight is True but k_proj_weight is None"
+ assert v_proj_weight is not None, "use_separate_proj_weight is True but v_proj_weight is None"
+ if in_proj_bias is None:
+ b_q = b_k = b_v = None
+ else:
+ b_q, b_k, b_v = in_proj_bias.chunk(3)
+ q, k, v = _in_projection(query, key, value, q_proj_weight, k_proj_weight, v_proj_weight, b_q, b_k, b_v)
+ if(cache!=None):
+ if(cache["first_infer"]==1):
+ cache["k"][cache["stage"]]=k
+ # print(0,cache["k"].shape)
+ cache["v"][cache["stage"]]=v
+ else:###12个layer每个都要留自己的cache_kv
+ # print(1,cache["k"].shape)
+ cache["k"][cache["stage"]]=torch.cat([cache["k"][cache["stage"]],k],0)##本来时序是1,但是proj的时候可能transpose了所以时序到0维了
+ cache["v"][cache["stage"]]=torch.cat([cache["v"][cache["stage"]],v],0)
+ # print(2, cache["k"].shape)
+ src_len = cache["k"][cache["stage"]].shape[0]
+ k=cache["k"][cache["stage"]]
+ v=cache["v"][cache["stage"]]
+ # if attn_mask is not None:
+ # attn_mask=attn_mask[-1:,]
+ # print(attn_mask.shape,attn_mask)
+ cache["stage"] = (cache["stage"] + 1) % cache["all_stage"]
+ # print(2333,cache)
+ # prep attention mask
+
+ attn_mask = _canonical_mask(
+ mask=attn_mask,
+ mask_name="attn_mask",
+ other_type=None,
+ other_name="",
+ target_type=q.dtype,
+ check_other=False,
+ )
+
+ if attn_mask is not None:
+ # ensure attn_mask's dim is 3
+ if attn_mask.dim() == 2:
+ correct_2d_size = (tgt_len, src_len)
+ if attn_mask.shape != correct_2d_size:
+ raise RuntimeError(f"The shape of the 2D attn_mask is {attn_mask.shape}, but should be {correct_2d_size}.")
+ attn_mask = attn_mask.unsqueeze(0)
+ elif attn_mask.dim() == 3:
+ correct_3d_size = (bsz * num_heads, tgt_len, src_len)
+ if attn_mask.shape != correct_3d_size:
+ raise RuntimeError(f"The shape of the 3D attn_mask is {attn_mask.shape}, but should be {correct_3d_size}.")
+ else:
+ raise RuntimeError(f"attn_mask's dimension {attn_mask.dim()} is not supported")
+
+ # add bias along batch dimension (currently second)
+ if bias_k is not None and bias_v is not None:
+ assert static_k is None, "bias cannot be added to static key."
+ assert static_v is None, "bias cannot be added to static value."
+ k = torch.cat([k, bias_k.repeat(1, bsz, 1)])
+ v = torch.cat([v, bias_v.repeat(1, bsz, 1)])
+ if attn_mask is not None:
+ attn_mask = pad(attn_mask, (0, 1))
+ if key_padding_mask is not None:
+ key_padding_mask = pad(key_padding_mask, (0, 1))
+ else:
+ assert bias_k is None
+ assert bias_v is None
+
+ #
+ # reshape q, k, v for multihead attention and make em batch first
+ #
+ q = q.view(tgt_len, bsz * num_heads, head_dim).transpose(0, 1)
+ if static_k is None:
+ k = k.view(k.shape[0], bsz * num_heads, head_dim).transpose(0, 1)
+ else:
+ # TODO finish disentangling control flow so we don't do in-projections when statics are passed
+ assert static_k.size(0) == bsz * num_heads, \
+ f"expecting static_k.size(0) of {bsz * num_heads}, but got {static_k.size(0)}"
+ assert static_k.size(2) == head_dim, \
+ f"expecting static_k.size(2) of {head_dim}, but got {static_k.size(2)}"
+ k = static_k
+ if static_v is None:
+ v = v.view(v.shape[0], bsz * num_heads, head_dim).transpose(0, 1)
+ else:
+ # TODO finish disentangling control flow so we don't do in-projections when statics are passed
+ assert static_v.size(0) == bsz * num_heads, \
+ f"expecting static_v.size(0) of {bsz * num_heads}, but got {static_v.size(0)}"
+ assert static_v.size(2) == head_dim, \
+ f"expecting static_v.size(2) of {head_dim}, but got {static_v.size(2)}"
+ v = static_v
+
+ # add zero attention along batch dimension (now first)
+ if add_zero_attn:
+ zero_attn_shape = (bsz * num_heads, 1, head_dim)
+ k = torch.cat([k, torch.zeros(zero_attn_shape, dtype=k.dtype, device=k.device)], dim=1)
+ v = torch.cat([v, torch.zeros(zero_attn_shape, dtype=v.dtype, device=v.device)], dim=1)
+ if attn_mask is not None:
+ attn_mask = pad(attn_mask, (0, 1))
+ if key_padding_mask is not None:
+ key_padding_mask = pad(key_padding_mask, (0, 1))
+
+ # update source sequence length after adjustments
+ src_len = k.size(1)
+
+ # merge key padding and attention masks
+ if key_padding_mask is not None:
+ assert key_padding_mask.shape == (bsz, src_len), \
+ f"expecting key_padding_mask shape of {(bsz, src_len)}, but got {key_padding_mask.shape}"
+ key_padding_mask = key_padding_mask.view(bsz, 1, 1, src_len). \
+ expand(-1, num_heads, -1, -1).reshape(bsz * num_heads, 1, src_len)
+ if attn_mask is None:
+ attn_mask = key_padding_mask
+ else:
+ attn_mask = attn_mask + key_padding_mask
+
+ # adjust dropout probability
+ if not training:
+ dropout_p = 0.0
+
+ #
+ # (deep breath) calculate attention and out projection
+ #
+
+ if need_weights:
+ B, Nt, E = q.shape
+ q_scaled = q / math.sqrt(E)
+
+ assert not (is_causal and attn_mask is None), "FIXME: is_causal not implemented for need_weights"
+
+ if attn_mask is not None:
+ attn_output_weights = torch.baddbmm(attn_mask, q_scaled, k.transpose(-2, -1))
+ else:
+ attn_output_weights = torch.bmm(q_scaled, k.transpose(-2, -1))
+ attn_output_weights = softmax(attn_output_weights, dim=-1)
+ if dropout_p > 0.0:
+ attn_output_weights = dropout(attn_output_weights, p=dropout_p)
+
+ attn_output = torch.bmm(attn_output_weights, v)
+
+ attn_output = attn_output.transpose(0, 1).contiguous().view(tgt_len * bsz, embed_dim)
+ attn_output = linear(attn_output, out_proj_weight, out_proj_bias)
+ attn_output = attn_output.view(tgt_len, bsz, attn_output.size(1))
+
+ # optionally average attention weights over heads
+ attn_output_weights = attn_output_weights.view(bsz, num_heads, tgt_len, src_len)
+ if average_attn_weights:
+ attn_output_weights = attn_output_weights.mean(dim=1)
+
+ if not is_batched:
+ # squeeze the output if input was unbatched
+ attn_output = attn_output.squeeze(1)
+ attn_output_weights = attn_output_weights.squeeze(0)
+ return attn_output, attn_output_weights
+ else:
+ # attn_mask can be either (L,S) or (N*num_heads, L, S)
+ # if attn_mask's shape is (1, L, S) we need to unsqueeze to (1, 1, L, S)
+ # in order to match the input for SDPA of (N, num_heads, L, S)
+ if attn_mask is not None:
+ if attn_mask.size(0) == 1 and attn_mask.dim() == 3:
+ attn_mask = attn_mask.unsqueeze(0)
+ else:
+ attn_mask = attn_mask.view(bsz, num_heads, -1, src_len)
+
+ q = q.view(bsz, num_heads, tgt_len, head_dim)
+ k = k.view(bsz, num_heads, src_len, head_dim)
+ v = v.view(bsz, num_heads, src_len, head_dim)
+
+ attn_output = scaled_dot_product_attention(q, k, v, attn_mask, dropout_p, is_causal)
+ attn_output = attn_output.permute(2, 0, 1, 3).contiguous().view(bsz * tgt_len, embed_dim)
+
+ attn_output = linear(attn_output, out_proj_weight, out_proj_bias)
+ attn_output = attn_output.view(tgt_len, bsz, attn_output.size(1))
+ if not is_batched:
+ # squeeze the output if input was unbatched
+ attn_output = attn_output.squeeze(1)
+ return attn_output, None
diff --git a/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/AR/modules/scaling.py b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/AR/modules/scaling.py
new file mode 100644
index 0000000000000000000000000000000000000000..ec31d616b18dec97962fb55dbc5c264cc1c3c39a
--- /dev/null
+++ b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/AR/modules/scaling.py
@@ -0,0 +1,319 @@
+# Copyright 2022 Xiaomi Corp. (authors: Daniel Povey)
+#
+# See ../../../../LICENSE for clarification regarding multiple authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+import math
+import random
+from typing import Optional
+from typing import Tuple
+from typing import Union
+
+import torch
+import torch.nn as nn
+from torch import Tensor
+
+
+class DoubleSwishFunction(torch.autograd.Function):
+ """
+ double_swish(x) = x * torch.sigmoid(x-1)
+ This is a definition, originally motivated by its close numerical
+ similarity to swish(swish(x)), where swish(x) = x * sigmoid(x).
+
+ Memory-efficient derivative computation:
+ double_swish(x) = x * s, where s(x) = torch.sigmoid(x-1)
+ double_swish'(x) = d/dx double_swish(x) = x * s'(x) + x' * s(x) = x * s'(x) + s(x).
+ Now, s'(x) = s(x) * (1-s(x)).
+ double_swish'(x) = x * s'(x) + s(x).
+ = x * s(x) * (1-s(x)) + s(x).
+ = double_swish(x) * (1-s(x)) + s(x)
+ ... so we just need to remember s(x) but not x itself.
+ """
+
+ @staticmethod
+ def forward(ctx, x: Tensor) -> Tensor:
+ requires_grad = x.requires_grad
+ x_dtype = x.dtype
+ if x.dtype == torch.float16:
+ x = x.to(torch.float32)
+
+ s = torch.sigmoid(x - 1.0)
+ y = x * s
+
+ if requires_grad:
+ deriv = y * (1 - s) + s
+ # notes on derivative of x * sigmoid(x - 1):
+ # https://www.wolframalpha.com/input?i=d%2Fdx+%28x+*+sigmoid%28x-1%29%29
+ # min \simeq -0.043638. Take floor as -0.043637 so it's a lower bund
+ # max \simeq 1.1990. Take ceil to be 1.2 so it's an upper bound.
+ # the combination of "+ torch.rand_like(deriv)" and casting to torch.uint8 (which
+ # floors), should be expectation-preserving.
+ floor = -0.043637
+ ceil = 1.2
+ d_scaled = (deriv - floor) * (255.0 / (ceil - floor)
+ ) + torch.rand_like(deriv)
+ if __name__ == "__main__":
+ # for self-testing only.
+ assert d_scaled.min() >= 0.0
+ assert d_scaled.max() < 256.0
+ d_int = d_scaled.to(torch.uint8)
+ ctx.save_for_backward(d_int)
+ if x.dtype == torch.float16 or torch.is_autocast_enabled():
+ y = y.to(torch.float16)
+ return y
+
+ @staticmethod
+ def backward(ctx, y_grad: Tensor) -> Tensor:
+ (d, ) = ctx.saved_tensors
+ # the same constants as used in forward pass.
+ floor = -0.043637
+ ceil = 1.2
+ d = d * ((ceil - floor) / 255.0) + floor
+ return y_grad * d
+
+
+class DoubleSwish(torch.nn.Module):
+ def forward(self, x: Tensor) -> Tensor:
+ """Return double-swish activation function which is an approximation to Swish(Swish(x)),
+ that we approximate closely with x * sigmoid(x-1).
+ """
+ if torch.jit.is_scripting() or torch.jit.is_tracing():
+ return x * torch.sigmoid(x - 1.0)
+ return DoubleSwishFunction.apply(x)
+
+
+class ActivationBalancerFunction(torch.autograd.Function):
+ @staticmethod
+ def forward(
+ ctx,
+ x: Tensor,
+ scale_factor: Tensor,
+ sign_factor: Optional[Tensor],
+ channel_dim: int, ) -> Tensor:
+ if channel_dim < 0:
+ channel_dim += x.ndim
+ ctx.channel_dim = channel_dim
+ xgt0 = x > 0
+ if sign_factor is None:
+ ctx.save_for_backward(xgt0, scale_factor)
+ else:
+ ctx.save_for_backward(xgt0, scale_factor, sign_factor)
+ return x
+
+ @staticmethod
+ def backward(ctx, x_grad: Tensor) -> Tuple[Tensor, None, None, None]:
+ if len(ctx.saved_tensors) == 3:
+ xgt0, scale_factor, sign_factor = ctx.saved_tensors
+ for _ in range(ctx.channel_dim, x_grad.ndim - 1):
+ scale_factor = scale_factor.unsqueeze(-1)
+ sign_factor = sign_factor.unsqueeze(-1)
+ factor = sign_factor + scale_factor * (xgt0.to(x_grad.dtype) - 0.5)
+ else:
+ xgt0, scale_factor = ctx.saved_tensors
+ for _ in range(ctx.channel_dim, x_grad.ndim - 1):
+ scale_factor = scale_factor.unsqueeze(-1)
+ factor = scale_factor * (xgt0.to(x_grad.dtype) - 0.5)
+ neg_delta_grad = x_grad.abs() * factor
+ return (x_grad - neg_delta_grad, None, None, None, )
+
+
+def _compute_scale_factor(
+ x: Tensor,
+ channel_dim: int,
+ min_abs: float,
+ max_abs: float,
+ gain_factor: float,
+ max_factor: float, ) -> Tensor:
+ if channel_dim < 0:
+ channel_dim += x.ndim
+ sum_dims = [d for d in range(x.ndim) if d != channel_dim]
+ x_abs_mean = torch.mean(x.abs(), dim=sum_dims).to(torch.float32)
+
+ if min_abs == 0.0:
+ below_threshold = 0.0
+ else:
+ # below_threshold is 0 if x_abs_mean > min_abs, can be at most max_factor if
+ # x_abs)_mean , min_abs.
+ below_threshold = (
+ (min_abs - x_abs_mean) * (gain_factor / min_abs)).clamp(
+ min=0, max=max_factor)
+
+ above_threshold = ((x_abs_mean - max_abs) * (gain_factor / max_abs)).clamp(
+ min=0, max=max_factor)
+
+ return below_threshold - above_threshold
+
+
+def _compute_sign_factor(
+ x: Tensor,
+ channel_dim: int,
+ min_positive: float,
+ max_positive: float,
+ gain_factor: float,
+ max_factor: float, ) -> Tensor:
+ if channel_dim < 0:
+ channel_dim += x.ndim
+ sum_dims = [d for d in range(x.ndim) if d != channel_dim]
+ proportion_positive = torch.mean((x > 0).to(torch.float32), dim=sum_dims)
+ if min_positive == 0.0:
+ factor1 = 0.0
+ else:
+ # 0 if proportion_positive >= min_positive, else can be
+ # as large as max_factor.
+ factor1 = ((min_positive - proportion_positive) *
+ (gain_factor / min_positive)).clamp_(
+ min=0, max=max_factor)
+
+ if max_positive == 1.0:
+ factor2 = 0.0
+ else:
+ # 0 if self.proportion_positive <= max_positive, else can be
+ # as large as -max_factor.
+ factor2 = ((proportion_positive - max_positive) *
+ (gain_factor / (1.0 - max_positive))).clamp_(
+ min=0, max=max_factor)
+ sign_factor = factor1 - factor2
+ # require min_positive != 0 or max_positive != 1:
+ assert not isinstance(sign_factor, float)
+ return sign_factor
+
+
+class ActivationBalancer(torch.nn.Module):
+ """
+ Modifies the backpropped derivatives of a function to try to encourage, for
+ each channel, that it is positive at least a proportion `threshold` of the
+ time. It does this by multiplying negative derivative values by up to
+ (1+max_factor), and positive derivative values by up to (1-max_factor),
+ interpolated from 1 at the threshold to those extremal values when none
+ of the inputs are positive.
+
+ Args:
+ num_channels: the number of channels
+ channel_dim: the dimension/axis corresponding to the channel, e.g.
+ -1, 0, 1, 2; will be interpreted as an offset from x.ndim if negative.
+ min_positive: the minimum, per channel, of the proportion of the time
+ that (x > 0), below which we start to modify the derivatives.
+ max_positive: the maximum, per channel, of the proportion of the time
+ that (x > 0), above which we start to modify the derivatives.
+ max_factor: the maximum factor by which we modify the derivatives for
+ either the sign constraint or the magnitude constraint;
+ e.g. with max_factor=0.02, the the derivatives would be multiplied by
+ values in the range [0.98..1.02].
+ sign_gain_factor: determines the 'gain' with which we increase the
+ change in gradient once the constraints on min_positive and max_positive
+ are violated.
+ scale_gain_factor: determines the 'gain' with which we increase the
+ change in gradient once the constraints on min_abs and max_abs
+ are violated.
+ min_abs: the minimum average-absolute-value difference from the mean
+ value per channel, which we allow, before we start to modify
+ the derivatives to prevent this.
+ max_abs: the maximum average-absolute-value difference from the mean
+ value per channel, which we allow, before we start to modify
+ the derivatives to prevent this.
+ min_prob: determines the minimum probability with which we modify the
+ gradients for the {min,max}_positive and {min,max}_abs constraints,
+ on each forward(). This is done randomly to prevent all layers
+ from doing it at the same time. Early in training we may use
+ higher probabilities than this; it will decay to this value.
+ """
+
+ def __init__(
+ self,
+ num_channels: int,
+ channel_dim: int,
+ min_positive: float=0.05,
+ max_positive: float=0.95,
+ max_factor: float=0.04,
+ sign_gain_factor: float=0.01,
+ scale_gain_factor: float=0.02,
+ min_abs: float=0.2,
+ max_abs: float=100.0,
+ min_prob: float=0.1, ):
+ super(ActivationBalancer, self).__init__()
+ self.num_channels = num_channels
+ self.channel_dim = channel_dim
+ self.min_positive = min_positive
+ self.max_positive = max_positive
+ self.max_factor = max_factor
+ self.min_abs = min_abs
+ self.max_abs = max_abs
+ self.min_prob = min_prob
+ self.sign_gain_factor = sign_gain_factor
+ self.scale_gain_factor = scale_gain_factor
+
+ # count measures how many times the forward() function has been called.
+ # We occasionally sync this to a tensor called `count`, that exists to
+ # make sure it is synced to disk when we load and save the model.
+ self.cpu_count = 0
+ self.register_buffer("count", torch.tensor(0, dtype=torch.int64))
+
+ def forward(self, x: Tensor) -> Tensor:
+ if (torch.jit.is_scripting() or not x.requires_grad or
+ torch.jit.is_tracing()):
+ return _no_op(x)
+
+ count = self.cpu_count
+ self.cpu_count += 1
+
+ if random.random() < 0.01:
+ # Occasionally sync self.cpu_count with self.count.
+ # count affects the decay of 'prob'. don't do this on every iter,
+ # because syncing with the GPU is slow.
+ self.cpu_count = max(self.cpu_count, self.count.item())
+ self.count.fill_(self.cpu_count)
+
+ # the prob of doing some work exponentially decreases from 0.5 till it hits
+ # a floor at min_prob (==0.1, by default)
+ prob = max(self.min_prob, 0.5**(1 + (count / 4000.0)))
+
+ if random.random() < prob:
+ sign_gain_factor = 0.5
+ if self.min_positive != 0.0 or self.max_positive != 1.0:
+ sign_factor = _compute_sign_factor(
+ x,
+ self.channel_dim,
+ self.min_positive,
+ self.max_positive,
+ gain_factor=self.sign_gain_factor / prob,
+ max_factor=self.max_factor, )
+ else:
+ sign_factor = None
+
+ scale_factor = _compute_scale_factor(
+ x.detach(),
+ self.channel_dim,
+ min_abs=self.min_abs,
+ max_abs=self.max_abs,
+ gain_factor=self.scale_gain_factor / prob,
+ max_factor=self.max_factor, )
+ return ActivationBalancerFunction.apply(
+ x,
+ scale_factor,
+ sign_factor,
+ self.channel_dim, )
+ else:
+ return _no_op(x)
+
+
+def BalancedDoubleSwish(d_model, channel_dim=-1, max_abs=10.0,
+ min_prob=0.25) -> nn.Sequential:
+ """
+ ActivationBalancer -> DoubleSwish
+ """
+ balancer = ActivationBalancer(
+ d_model, channel_dim=channel_dim, max_abs=max_abs, min_prob=min_prob)
+ return nn.Sequential(
+ balancer,
+ DoubleSwish(), )
diff --git a/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/AR/modules/transformer.py b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/AR/modules/transformer.py
new file mode 100644
index 0000000000000000000000000000000000000000..04f0b1bea152c8f6cd5c6fc0ea60058691e0947e
--- /dev/null
+++ b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/AR/modules/transformer.py
@@ -0,0 +1,347 @@
+# modified from https://github.com/lifeiteng/vall-e/blob/main/valle/modules/transformer.py
+import copy
+import numbers
+from functools import partial
+from typing import Any
+from typing import Callable
+from typing import List
+from typing import Optional
+from typing import Tuple
+from typing import Union
+
+import torch
+from AR.modules.activation import MultiheadAttention
+from AR.modules.scaling import BalancedDoubleSwish
+from torch import nn
+from torch import Tensor
+from torch.nn import functional as F
+
+_shape_t = Union[int, List[int], torch.Size]
+
+
+class LayerNorm(nn.Module):
+ __constants__ = ["normalized_shape", "eps", "elementwise_affine"]
+ normalized_shape: Tuple[int, ...]
+ eps: float
+ elementwise_affine: bool
+
+ def __init__(
+ self,
+ normalized_shape: _shape_t,
+ eps: float=1e-5,
+ elementwise_affine: bool=True,
+ device=None,
+ dtype=None, ) -> None:
+ factory_kwargs = {"device": device, "dtype": dtype}
+ super(LayerNorm, self).__init__()
+ if isinstance(normalized_shape, numbers.Integral):
+ # mypy error: incompatible types in assignment
+ normalized_shape = (normalized_shape, ) # type: ignore[assignment]
+ self.normalized_shape = tuple(
+ normalized_shape) # type: ignore[arg-type]
+ self.eps = eps
+ self.elementwise_affine = elementwise_affine
+ if self.elementwise_affine:
+ self.weight = nn.Parameter(
+ torch.empty(self.normalized_shape, **factory_kwargs))
+ self.bias = nn.Parameter(
+ torch.empty(self.normalized_shape, **factory_kwargs))
+ else:
+ self.register_parameter("weight", None)
+ self.register_parameter("bias", None)
+
+ self.reset_parameters()
+
+ def reset_parameters(self) -> None:
+ if self.elementwise_affine:
+ nn.init.ones_(self.weight)
+ nn.init.zeros_(self.bias)
+
+ def forward(self, input: Tensor, embedding: Any=None) -> Tensor:
+ if isinstance(input, tuple):
+ input, embedding = input
+ return (F.layer_norm(
+ input,
+ self.normalized_shape,
+ self.weight,
+ self.bias,
+ self.eps, ), embedding, )
+
+ assert embedding is None
+ return F.layer_norm(input, self.normalized_shape, self.weight,
+ self.bias, self.eps)
+
+ def extra_repr(self) -> str:
+ return (
+ "{normalized_shape}, eps={eps}, "
+ "elementwise_affine={elementwise_affine}".format(**self.__dict__))
+
+
+class IdentityNorm(nn.Module):
+ def __init__(
+ self,
+ d_model: int,
+ eps: float=1e-5,
+ device=None,
+ dtype=None, ) -> None:
+ super(IdentityNorm, self).__init__()
+
+ def forward(self, input: Tensor, embedding: Any=None) -> Tensor:
+ if isinstance(input, tuple):
+ return input
+
+ assert embedding is None
+ return input
+
+
+class TransformerEncoder(nn.Module):
+ r"""TransformerEncoder is a stack of N encoder layers. Users can build the
+ BERT(https://arxiv.org/abs/1810.04805) model with corresponding parameters.
+
+ Args:
+ encoder_layer: an instance of the TransformerEncoderLayer() class (required).
+ num_layers: the number of sub-encoder-layers in the encoder (required).
+ norm: the layer normalization component (optional).
+ enable_nested_tensor: if True, input will automatically convert to nested tensor
+ (and convert back on output). This will improve the overall performance of
+ TransformerEncoder when padding rate is high. Default: ``True`` (enabled).
+
+ Examples::
+ >>> encoder_layer = TransformerEncoderLayer(d_model=512, nhead=8)
+ >>> transformer_encoder = TransformerEncoder(encoder_layer, num_layers=6)
+ >>> src = torch.rand(10, 32, 512)
+ >>> out = transformer_encoder(src)
+ """
+ __constants__ = ["norm"]
+
+ def __init__(self, encoder_layer, num_layers, norm=None):
+ super(TransformerEncoder, self).__init__()
+ self.layers = _get_clones(encoder_layer, num_layers)
+ self.num_layers = num_layers
+ self.norm = norm
+
+ def forward(
+ self,
+ src: Tensor,
+ mask: Optional[Tensor]=None,
+ src_key_padding_mask: Optional[Tensor]=None,
+ return_layer_states: bool=False,cache=None ) -> Tensor:
+ r"""Pass the input through the encoder layers in turn.
+
+ Args:
+ src: the sequence to the encoder (required).
+ mask: the mask for the src sequence (optional).
+ src_key_padding_mask: the mask for the src keys per batch (optional).
+ return_layer_states: return layers' state (optional).
+
+ Shape:
+ see the docs in Transformer class.
+ """
+ if return_layer_states:
+ layer_states = [] # layers' output
+ output = src
+ for mod in self.layers:
+ output = mod(
+ output,
+ src_mask=mask,
+ src_key_padding_mask=src_key_padding_mask, cache=cache)
+ layer_states.append(output[0])
+
+ if self.norm is not None:
+ output = self.norm(output)
+
+ return layer_states, output
+
+ output = src
+ for mod in self.layers:
+ output = mod(output,
+ src_mask=mask,
+ src_key_padding_mask=src_key_padding_mask, cache=cache)
+
+ if self.norm is not None:
+ output = self.norm(output)
+
+ return output
+
+
+class TransformerEncoderLayer(nn.Module):
+ __constants__ = ["batch_first", "norm_first"]
+
+ def __init__(
+ self,
+ d_model: int,
+ nhead: int,
+ dim_feedforward: int=2048,
+ dropout: float=0.1,
+ activation: Union[str, Callable[[Tensor], Tensor]]=F.relu,
+ batch_first: bool=False,
+ norm_first: bool=False,
+ device=None,
+ dtype=None,
+ linear1_self_attention_cls: nn.Module=nn.Linear,
+ linear2_self_attention_cls: nn.Module=nn.Linear,
+ linear1_feedforward_cls: nn.Module=nn.Linear,
+ linear2_feedforward_cls: nn.Module=nn.Linear,
+ layer_norm_cls: nn.Module=LayerNorm,
+ layer_norm_eps: float=1e-5,
+ adaptive_layer_norm=False, ) -> None:
+ factory_kwargs = {"device": device, "dtype": dtype}
+ super(TransformerEncoderLayer, self).__init__()
+ # print(233333333333,d_model,nhead)
+ # import os
+ # os._exit(2333333)
+ self.self_attn = MultiheadAttention(
+ d_model,#512 16
+ nhead,
+ dropout=dropout,
+ batch_first=batch_first,
+ linear1_cls=linear1_self_attention_cls,
+ linear2_cls=linear2_self_attention_cls,
+ **factory_kwargs, )
+
+ # Implementation of Feedforward model
+ self.linear1 = linear1_feedforward_cls(d_model, dim_feedforward,
+ **factory_kwargs)
+ self.dropout = nn.Dropout(dropout)
+ self.linear2 = linear2_feedforward_cls(dim_feedforward, d_model,
+ **factory_kwargs)
+
+ self.norm_first = norm_first
+ self.dropout1 = nn.Dropout(dropout)
+ self.dropout2 = nn.Dropout(dropout)
+
+ # Legacy string support for activation function.
+ if isinstance(activation, str):
+ activation = _get_activation_fn(activation)
+ elif isinstance(activation, partial):
+ activation = activation(d_model)
+ elif activation == BalancedDoubleSwish:
+ activation = BalancedDoubleSwish(d_model)
+
+ # # We can't test self.activation in forward() in TorchScript,
+ # # so stash some information about it instead.
+ # if activation is F.relu or isinstance(activation, torch.nn.ReLU):
+ # self.activation_relu_or_gelu = 1
+ # elif activation is F.gelu or isinstance(activation, torch.nn.GELU):
+ # self.activation_relu_or_gelu = 2
+ # else:
+ # self.activation_relu_or_gelu = 0
+ self.activation = activation
+
+ norm1 = layer_norm_cls(d_model, eps=layer_norm_eps, **factory_kwargs)
+ if layer_norm_cls == IdentityNorm:
+ norm2 = BalancedBasicNorm(
+ d_model, eps=layer_norm_eps, **factory_kwargs)
+ else:
+ norm2 = layer_norm_cls(
+ d_model, eps=layer_norm_eps, **factory_kwargs)
+
+ if adaptive_layer_norm:
+ self.norm1 = AdaptiveLayerNorm(d_model, norm1)
+ self.norm2 = AdaptiveLayerNorm(d_model, norm2)
+ else:
+ self.norm1 = norm1
+ self.norm2 = norm2
+
+ def __setstate__(self, state):
+ super(TransformerEncoderLayer, self).__setstate__(state)
+ if not hasattr(self, "activation"):
+ self.activation = F.relu
+
+ def forward(
+ self,
+ src: Tensor,
+ src_mask: Optional[Tensor]=None,
+ src_key_padding_mask: Optional[Tensor]=None,cache=None ) -> Tensor:
+ r"""Pass the input through the encoder layer.
+
+ Args:
+ src: the sequence to the encoder layer (required).
+ src_mask: the mask for the src sequence (optional).
+ src_key_padding_mask: the mask for the src keys per batch (optional).
+
+ Shape:
+ see the docs in Transformer class.
+ """
+ x, stage_embedding = src, None
+ is_src_tuple = False
+ if isinstance(src, tuple):
+ x, stage_embedding = src
+ is_src_tuple = True
+
+ if src_key_padding_mask is not None:
+ _skpm_dtype = src_key_padding_mask.dtype
+ if _skpm_dtype != torch.bool and not torch.is_floating_point(
+ src_key_padding_mask):
+ raise AssertionError(
+ "only bool and floating types of key_padding_mask are supported"
+ )
+
+ if self.norm_first:
+ x = x + self._sa_block(
+ self.norm1(x, stage_embedding),
+ src_mask,
+ src_key_padding_mask,cache=cache )
+ x = x + self._ff_block(self.norm2(x, stage_embedding))
+ else:
+ x = self.norm1(
+ x + self._sa_block(x, src_mask, src_key_padding_mask,cache=cache),
+ stage_embedding, )
+ x = self.norm2(x + self._ff_block(x), stage_embedding)
+
+ if is_src_tuple:
+ return (x, stage_embedding)
+ return x
+
+ # self-attention block
+ def _sa_block(
+ self,
+ x: Tensor,
+ attn_mask: Optional[Tensor],
+ key_padding_mask: Optional[Tensor],cache=None ) -> Tensor:
+ # print(x.shape,attn_mask.shape,key_padding_mask)
+ #torch.Size([1, 188, 512]) torch.Size([188, 188]) None
+ # import os
+ # os._exit(23333)
+ x = self.self_attn(
+ x,
+ x,
+ x,
+ attn_mask=attn_mask,
+ key_padding_mask=key_padding_mask,
+ need_weights=False,cache=cache )[0]
+ return self.dropout1(x)
+
+ # feed forward block
+ def _ff_block(self, x: Tensor) -> Tensor:
+ x = self.linear2(self.dropout(self.activation(self.linear1(x))))
+ return self.dropout2(x)
+
+
+class AdaptiveLayerNorm(nn.Module):
+ r"""Adaptive Layer Normalization"""
+
+ def __init__(self, d_model, norm) -> None:
+ super(AdaptiveLayerNorm, self).__init__()
+ self.project_layer = nn.Linear(d_model, 2 * d_model)
+ self.norm = norm
+ self.d_model = d_model
+ self.eps = self.norm.eps
+
+ def forward(self, input: Tensor, embedding: Tensor=None) -> Tensor:
+ if isinstance(input, tuple):
+ input, embedding = input
+ weight, bias = torch.split(
+ self.project_layer(embedding),
+ split_size_or_sections=self.d_model,
+ dim=-1, )
+ return (weight * self.norm(input) + bias, embedding)
+
+ weight, bias = torch.split(
+ self.project_layer(embedding),
+ split_size_or_sections=self.d_model,
+ dim=-1, )
+ return weight * self.norm(input) + bias
+
+def _get_clones(module, N):
+ return nn.ModuleList([copy.deepcopy(module) for i in range(N)])
diff --git a/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/AR/text_processing/__init__.py b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/AR/text_processing/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/AR/text_processing/phonemizer.py b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/AR/text_processing/phonemizer.py
new file mode 100644
index 0000000000000000000000000000000000000000..83ecfb76fa0fe1ab8a6e78380c0534eee165b4db
--- /dev/null
+++ b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/AR/text_processing/phonemizer.py
@@ -0,0 +1,80 @@
+# modified from https://github.com/feng-yufei/shared_debugging_code/blob/main/text_processing/phonemizer.py
+import itertools
+import re
+from typing import Dict
+from typing import List
+
+import regex
+from gruut import sentences
+from gruut.const import Sentence
+from gruut.const import Word
+from AR.text_processing.symbols import SYMBOL_TO_ID
+
+
+class GruutPhonemizer:
+ def __init__(self, language: str):
+ self._phonemizer = sentences
+ self.lang = language
+ self.symbol_to_id = SYMBOL_TO_ID
+ self._special_cases_dict: Dict[str] = {
+ r"\.\.\.": "... ",
+ ";": "; ",
+ ":": ": ",
+ ",": ", ",
+ r"\.": ". ",
+ "!": "! ",
+ r"\?": "? ",
+ "—": "—",
+ "…": "… ",
+ "«": "«",
+ "»": "»"
+ }
+ self._punctuation_regexp: str = rf"([{''.join(self._special_cases_dict.keys())}])"
+
+ def _normalize_punctuation(self, text: str) -> str:
+ text = regex.sub(fr"\pZ+{self._punctuation_regexp}", r"\1", text)
+ text = regex.sub(fr"{self._punctuation_regexp}(\pL)", r"\1 \2", text)
+ text = regex.sub(r"\pZ+", r" ", text)
+ return text.strip()
+
+ def _convert_punctuation(self, word: Word) -> str:
+ if not word.phonemes:
+ return ''
+ if word.phonemes[0] in ['‖', '|']:
+ return word.text.strip()
+
+ phonemes = ''.join(word.phonemes)
+ # remove modifier characters ˈˌː with regex
+ phonemes = re.sub(r'[ˈˌː͡]', '', phonemes)
+ return phonemes.strip()
+
+ def phonemize(self, text: str, espeak: bool=False) -> str:
+ text_to_phonemize: str = self._normalize_punctuation(text)
+ sents: List[Sentence] = [
+ sent
+ for sent in self._phonemizer(
+ text_to_phonemize, lang="en-us", espeak=espeak)
+ ]
+ words: List[str] = [
+ self._convert_punctuation(word) for word in itertools.chain(*sents)
+ ]
+ return ' '.join(words)
+
+ def transform(self, phonemes):
+ # convert phonemes to ids
+ # dictionary is in symbols.py
+ return [
+ self.symbol_to_id[p] for p in phonemes
+ if p in self.symbol_to_id.keys()
+ ]
+
+
+if __name__ == "__main__":
+ phonemizer = GruutPhonemizer("en-us")
+ # text -> IPA
+ phonemes = phonemizer.phonemize("Hello, wor-ld ?")
+ print("phonemes:", phonemes)
+ print("len(phonemes):", len(phonemes))
+ phoneme_ids = phonemizer.transform(phonemes)
+ print("phoneme_ids:", phoneme_ids)
+ print("len(phoneme_ids):", len(phoneme_ids))
diff --git a/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/AR/text_processing/symbols.py b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/AR/text_processing/symbols.py
new file mode 100644
index 0000000000000000000000000000000000000000..6bc9a0c3eb77594b10670eb8caca7016cbbe9fdf
--- /dev/null
+++ b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/AR/text_processing/symbols.py
@@ -0,0 +1,9 @@
+# modified from https://github.com/feng-yufei/shared_debugging_code/blob/main/text_processing/symbols.py
+PAD = '_'
+PUNCTUATION = ';:,.!?¡¿—…"«»“” '
+LETTERS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
+IPA_LETTERS = "ɑɐɒæɓʙβɔɕçɗɖðʤəɘɚɛɜɝɞɟʄɡɠɢʛɦɧħɥʜɨɪʝɭɬɫɮʟɱɯɰŋɳɲɴøɵɸθœɶʘɹɺɾɻʀʁɽʂʃʈʧʉʊʋⱱʌɣɤʍχʎʏʑʐʒʔʡʕʢǀǁǂǃˈˌːˑʼʴʰʱʲʷˠˤ˞↓↑→↗↘'̩'ᵻ"
+SYMBOLS = [PAD] + list(PUNCTUATION) + list(LETTERS) + list(IPA_LETTERS)
+SPACE_ID = SYMBOLS.index(" ")
+SYMBOL_TO_ID = {s: i for i, s in enumerate(SYMBOLS)}
+ID_TO_SYMBOL = {i: s for i, s in enumerate(SYMBOLS)}
diff --git a/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/AR/utils/__init__.py b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/AR/utils/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..c2eaf61adcfee96d6e7ec8fd70a7603e18afb567
--- /dev/null
+++ b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/AR/utils/__init__.py
@@ -0,0 +1,37 @@
+import re
+
+
+def str2bool(str):
+ return True if str.lower() == 'true' else False
+
+
+def get_newest_ckpt(string_list):
+ # 定义一个正则表达式模式,用于匹配字符串中的数字
+ pattern = r'epoch=(\d+)-step=(\d+)\.ckpt'
+
+ # 使用正则表达式提取每个字符串中的数字信息,并创建一个包含元组的列表
+ extracted_info = []
+ for string in string_list:
+ match = re.match(pattern, string)
+ if match:
+ epoch = int(match.group(1))
+ step = int(match.group(2))
+ extracted_info.append((epoch, step, string))
+ # 按照 epoch 后面的数字和 step 后面的数字进行排序
+ sorted_info = sorted(
+ extracted_info, key=lambda x: (x[0], x[1]), reverse=True)
+ # 获取最新的 ckpt 文件名
+ newest_ckpt = sorted_info[0][2]
+ return newest_ckpt
+
+
+# 文本存在且不为空时 return True
+def check_txt_file(file_path):
+ try:
+ with open(file_path, 'r') as file:
+ text = file.readline().strip()
+ assert text.strip() != ''
+ return text
+ except Exception:
+ return False
+ return False
diff --git a/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/AR/utils/initialize.py b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/AR/utils/initialize.py
new file mode 100644
index 0000000000000000000000000000000000000000..17ff9f92e51c8941973139d6e34d4a1c7cd8daaa
--- /dev/null
+++ b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/AR/utils/initialize.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+"""Initialize modules for espnet2 neural networks."""
+import torch
+from typeguard import check_argument_types
+
+
+def initialize(model: torch.nn.Module, init: str):
+ """Initialize weights of a neural network module.
+
+ Parameters are initialized using the given method or distribution.
+
+ Custom initialization routines can be implemented into submodules
+ as function `espnet_initialization_fn` within the custom module.
+
+ Args:
+ model: Target.
+ init: Method of initialization.
+ """
+ assert check_argument_types()
+ print("init with", init)
+
+ # weight init
+ for p in model.parameters():
+ if p.dim() > 1:
+ if init == "xavier_uniform":
+ torch.nn.init.xavier_uniform_(p.data)
+ elif init == "xavier_normal":
+ torch.nn.init.xavier_normal_(p.data)
+ elif init == "kaiming_uniform":
+ torch.nn.init.kaiming_uniform_(p.data, nonlinearity="relu")
+ elif init == "kaiming_normal":
+ torch.nn.init.kaiming_normal_(p.data, nonlinearity="relu")
+ else:
+ raise ValueError("Unknown initialization: " + init)
+ # bias init
+ for name, p in model.named_parameters():
+ if ".bias" in name and p.dim() == 1:
+ p.data.zero_()
diff --git a/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/AR/utils/io.py b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/AR/utils/io.py
new file mode 100644
index 0000000000000000000000000000000000000000..24f1be625d1c767fe68c2f348b1d04011cd82699
--- /dev/null
+++ b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/AR/utils/io.py
@@ -0,0 +1,32 @@
+import sys
+
+import torch
+import yaml
+
+
+def load_yaml_config(path):
+ with open(path) as f:
+ config = yaml.full_load(f)
+ return config
+
+
+def save_config_to_yaml(config, path):
+ assert path.endswith('.yaml')
+ with open(path, 'w') as f:
+ f.write(yaml.dump(config))
+ f.close()
+
+
+def write_args(args, path):
+ args_dict = dict((name, getattr(args, name)) for name in dir(args)
+ if not name.startswith('_'))
+ with open(path, 'a') as args_file:
+ args_file.write('==> torch version: {}\n'.format(torch.__version__))
+ args_file.write(
+ '==> cudnn version: {}\n'.format(torch.backends.cudnn.version()))
+ args_file.write('==> Cmd:\n')
+ args_file.write(str(sys.argv))
+ args_file.write('\n==> args:\n')
+ for k, v in sorted(args_dict.items()):
+ args_file.write(' %s: %s\n' % (str(k), str(v)))
+ args_file.close()
diff --git a/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/configs/s1.yaml b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/configs/s1.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5481b9b7531fc1cba043e7d410f44d1ff84c0eec
--- /dev/null
+++ b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/configs/s1.yaml
@@ -0,0 +1,31 @@
+train:
+ seed: 1234
+ epochs: 300
+ batch_size: 8
+ gradient_accumulation: 4
+ save_every_n_epoch: 1
+ precision: 16
+ gradient_clip: 1.0
+optimizer:
+ lr: 0.01
+ lr_init: 0.00001
+ lr_end: 0.0001
+ warmup_steps: 2000
+ decay_steps: 40000
+data:
+ max_eval_sample: 8
+ max_sec: 54
+ num_workers: 1
+ pad_val: 1024 # same with EOS in model
+model:
+ vocab_size: 1025
+ phoneme_vocab_size: 512
+ embedding_dim: 512
+ hidden_dim: 512
+ head: 16
+ linear_units: 2048
+ n_layer: 12
+ dropout: 0
+ EOS: 1024
+inference:
+ top_k: 5
\ No newline at end of file
diff --git a/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/configs/s1big.yaml b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/configs/s1big.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..3a17ae53d0ff4e1f37d4a1b3ba6fed9a52a5c9d6
--- /dev/null
+++ b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/configs/s1big.yaml
@@ -0,0 +1,31 @@
+train:
+ seed: 1234
+ epochs: 300
+ batch_size: 8
+ gradient_accumulation: 4
+ save_every_n_epoch: 1
+ precision: 16-mixed
+ gradient_clip: 1.0
+optimizer:
+ lr: 0.01
+ lr_init: 0.00001
+ lr_end: 0.0001
+ warmup_steps: 2000
+ decay_steps: 40000
+data:
+ max_eval_sample: 8
+ max_sec: 54
+ num_workers: 1
+ pad_val: 1024 # same with EOS in model
+model:
+ vocab_size: 1025
+ phoneme_vocab_size: 512
+ embedding_dim: 1024
+ hidden_dim: 1024
+ head: 16
+ linear_units: 2048
+ n_layer: 16
+ dropout: 0
+ EOS: 1024
+inference:
+ top_k: 5
\ No newline at end of file
diff --git a/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/configs/s1big2.yaml b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/configs/s1big2.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..1037fc7dc0aafffe7ec94da33f27cc744ff12bae
--- /dev/null
+++ b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/configs/s1big2.yaml
@@ -0,0 +1,31 @@
+train:
+ seed: 1234
+ epochs: 300
+ batch_size: 12
+ gradient_accumulation: 4
+ save_every_n_epoch: 1
+ precision: 16-mixed
+ gradient_clip: 1.0
+optimizer:
+ lr: 0.01
+ lr_init: 0.00001
+ lr_end: 0.0001
+ warmup_steps: 2000
+ decay_steps: 40000
+data:
+ max_eval_sample: 8
+ max_sec: 54
+ num_workers: 1
+ pad_val: 1024 # same with EOS in model
+model:
+ vocab_size: 1025
+ phoneme_vocab_size: 512
+ embedding_dim: 1024
+ hidden_dim: 1024
+ head: 16
+ linear_units: 2048
+ n_layer: 6
+ dropout: 0
+ EOS: 1024
+inference:
+ top_k: 5
\ No newline at end of file
diff --git a/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/configs/s1longer.yaml b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/configs/s1longer.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b238abd26291252be81eee2a9eb62479c5fd246b
--- /dev/null
+++ b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/configs/s1longer.yaml
@@ -0,0 +1,31 @@
+train:
+ seed: 1234
+ epochs: 20
+ batch_size: 8
+ save_every_n_epoch: 1
+ precision: 16-mixed
+ gradient_clip: 1.0
+optimizer:
+ lr: 0.01
+ lr_init: 0.00001
+ lr_end: 0.0001
+ warmup_steps: 2000
+ decay_steps: 40000
+data:
+ max_eval_sample: 8
+ max_sec: 54
+ num_workers: 4
+ pad_val: 1024 # same with EOS in model
+model:
+ vocab_size: 1025
+ phoneme_vocab_size: 512
+ embedding_dim: 512
+ hidden_dim: 512
+ head: 16
+ linear_units: 2048
+ n_layer: 24
+ dropout: 0
+ EOS: 1024
+ random_bert: 0
+inference:
+ top_k: 5
\ No newline at end of file
diff --git a/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/configs/s1mq.yaml b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/configs/s1mq.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..19aac92eb4096c9d7ef6339e4ca5ce68dcc600f6
--- /dev/null
+++ b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/configs/s1mq.yaml
@@ -0,0 +1,77 @@
+train:
+ seed: 1234
+ epochs: 100
+ batch_size: 6
+ gradient_accumulation: 4
+ save_every_n_epoch: 1
+ precision: 32
+ gradient_clip: 1.0
+optimizer:
+ lr: 0.01
+ lr_init: 0.00001
+ lr_end: 0.0001
+ warmup_steps: 2000
+ decay_steps: 40000
+data:
+ max_eval_sample: 8
+ max_sec: 40
+ num_workers: 1
+ pad_val: 1024 # same with EOS in model
+model:
+ saving_path: "ckpt/"
+ resume_checkpoint: null
+ vocoder_config_path: "quantizer/new_ckpt/config.json"
+ vocoder_ckpt_path: "quantizer/new_ckpt/g_00600000"
+ datadir: "/home/liweiche/GigaSpeech/wavs"
+ metapath: "/home/liweiche/GigaSpeech/train2.json"
+ val_metapath: "/home/liweiche/GigaSpeech/dev2.json"
+ sampledir: "logs/"
+ pretrained_path: null
+ lr: 0.0001
+ batch_size: 200.0
+ train_bucket_size: 8192
+ training_step: 800000
+ optim_flat_percent: 0.0
+ warmup_step: 50
+ adam_beta1: 0.9
+ adam_beta2: 0.98
+ ffd_size: 3072
+ hidden_size: 768
+ enc_nlayers: 6
+ dec_nlayers: 6
+ nheads: 12
+ ar_layer: 4
+ ar_ffd_size: 1024
+ ar_hidden_size: 256
+ ar_nheads: 4
+ aligner_softmax_temp: 1.0
+ layer_norm_eps: 0.00001
+ speaker_embed_dropout: 0.05
+ label_smoothing: 0.0
+ val_check_interval: 5000
+ check_val_every_n_epoch: 1
+ precision: "fp16"
+ nworkers: 16
+ distributed: true
+ accelerator: "ddp"
+ version: null
+ accumulate_grad_batches: 1
+ use_repetition_token: true
+ use_repetition_gating: false
+ repetition_penalty: 1.0
+ sampling_temperature: 1.0
+ top_k: -1
+ min_top_k: 3
+ top_p: 0.8
+ sample_num: 4
+ length_penalty_max_length: 15000
+ length_penalty_max_prob: 0.95
+ max_input_length: 2048
+ max_output_length: 2000
+ sample_rate: 16000
+ n_codes: 1024
+ n_cluster_groups: 1
+ phone_context_window: 4
+ phoneset_size: 1000
+inference:
+ top_k: 5
\ No newline at end of file
diff --git a/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/configs/s2.json b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/configs/s2.json
new file mode 100644
index 0000000000000000000000000000000000000000..e44e1eb7d9864a20c76d35201f1ce0b9b15f3e95
--- /dev/null
+++ b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/configs/s2.json
@@ -0,0 +1,90 @@
+{
+ "train": {
+ "log_interval": 100,
+ "eval_interval": 500,
+ "seed": 1234,
+ "epochs": 100,
+ "learning_rate": 0.0001,
+ "betas": [
+ 0.8,
+ 0.99
+ ],
+ "eps": 1e-09,
+ "batch_size": 32,
+ "fp16_run": true,
+ "lr_decay": 0.999875,
+ "segment_size": 20480,
+ "init_lr_ratio": 1,
+ "warmup_epochs": 0,
+ "c_mel": 45,
+ "c_kl": 1.0,
+ "text_low_lr_rate": 0.4
+ },
+ "data": {
+ "max_wav_value": 32768.0,
+ "sampling_rate": 32000,
+ "filter_length": 2048,
+ "hop_length": 640,
+ "win_length": 2048,
+ "n_mel_channels": 128,
+ "mel_fmin": 0.0,
+ "mel_fmax": null,
+ "add_blank": true,
+ "n_speakers": 300,
+ "cleaned_text": true
+ },
+ "model": {
+ "inter_channels": 192,
+ "hidden_channels": 192,
+ "filter_channels": 768,
+ "n_heads": 2,
+ "n_layers": 6,
+ "kernel_size": 3,
+ "p_dropout": 0.1,
+ "resblock": "1",
+ "resblock_kernel_sizes": [
+ 3,
+ 7,
+ 11
+ ],
+ "resblock_dilation_sizes": [
+ [
+ 1,
+ 3,
+ 5
+ ],
+ [
+ 1,
+ 3,
+ 5
+ ],
+ [
+ 1,
+ 3,
+ 5
+ ]
+ ],
+ "upsample_rates": [
+ 10,
+ 8,
+ 2,
+ 2,
+ 2
+ ],
+ "upsample_initial_channel": 512,
+ "upsample_kernel_sizes": [
+ 16,
+ 16,
+ 8,
+ 2,
+ 2
+ ],
+ "n_layers_q": 3,
+ "use_spectral_norm": false,
+ "gin_channels": 512,
+ "semantic_frame_rate": "25hz",
+ "freeze_quantizer": true
+ },
+ "s2_ckpt_dir": "logs/s2/big2k1",
+ "content_module": "cnhubert"
+}
\ No newline at end of file
diff --git a/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/configs/train.yaml b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/configs/train.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a61e90d4e2f25746f80c91a2a5aadece364e1906
--- /dev/null
+++ b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/configs/train.yaml
@@ -0,0 +1,32 @@
+gpu:
+ n_card: 1
+ n_process_per_card: 2
+io:
+ text_path: D:\RVC1006\GPT-SoVITS\GPT_SoVITS
+ save_every_n_epoch: 1
+ precision: 16-mixed
+ gradient_clip: 1.0
+optimizer:
+ lr: 0.01
+ lr_init: 0.00001
+ lr_end: 0.0001
+ warmup_steps: 2000
+ decay_steps: 40000
+data:
+ max_eval_sample: 8
+ max_sec: 54
+ num_workers: 1
+ pad_val: 1024 # same with EOS in model
+model:
+ vocab_size: 1025
+ phoneme_vocab_size: 512
+ embedding_dim: 512
+ hidden_dim: 512
+ head: 16
+ linear_units: 2048
+ n_layer: 24
+ dropout: 0
+ EOS: 1024
+ random_bert: 0
+inference:
+ top_k: 5
\ No newline at end of file
diff --git a/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/feature_extractor/__init__.py b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/feature_extractor/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..79aa9294fec8fd823afae7d1a18e3279533dd7cf
--- /dev/null
+++ b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/feature_extractor/__init__.py
@@ -0,0 +1,6 @@
+from . import cnhubert, whisper_enc
+
+content_module_map = {
+ 'cnhubert': cnhubert,
+ 'whisper': whisper_enc
+}
\ No newline at end of file
diff --git a/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/feature_extractor/cnhubert.py b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/feature_extractor/cnhubert.py
new file mode 100644
index 0000000000000000000000000000000000000000..048dc85c8093cd849232b34c8b059cf6040a4cc0
--- /dev/null
+++ b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/feature_extractor/cnhubert.py
@@ -0,0 +1,97 @@
+import time
+
+import librosa
+import torch
+import torch.nn.functional as F
+import soundfile as sf
+import logging
+
+logging.getLogger("numba").setLevel(logging.WARNING)
+
+from transformers import (
+ Wav2Vec2FeatureExtractor,
+ HubertModel,
+ Wav2Vec2Model,
+)
+
+import utils
+import torch.nn as nn
+
+cnhubert_base_path=None
+class CNHubert(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.model = HubertModel.from_pretrained(cnhubert_base_path)
+ self.feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(cnhubert_base_path)
+ def forward(self, x):
+ input_values = self.feature_extractor(x, return_tensors="pt", sampling_rate=16000).input_values.to(x.device)
+ feats = self.model(input_values)["last_hidden_state"]
+ return feats
+
+# class CNHubertLarge(nn.Module):
+# def __init__(self):
+# super().__init__()
+# self.model = HubertModel.from_pretrained("/data/docker/liujing04/gpt-vits/chinese-hubert-large")
+# self.feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained("/data/docker/liujing04/gpt-vits/chinese-hubert-large")
+# def forward(self, x):
+# input_values = self.feature_extractor(x, return_tensors="pt", sampling_rate=16000).input_values.to(x.device)
+# feats = self.model(input_values)["last_hidden_state"]
+# return feats
+#
+# class CVec(nn.Module):
+# def __init__(self):
+# super().__init__()
+# self.model = HubertModel.from_pretrained("/data/docker/liujing04/vc-webui-big/hubert_base")
+# self.feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained("/data/docker/liujing04/vc-webui-big/hubert_base")
+# def forward(self, x):
+# input_values = self.feature_extractor(x, return_tensors="pt", sampling_rate=16000).input_values.to(x.device)
+# feats = self.model(input_values)["last_hidden_state"]
+# return feats
+#
+# class cnw2v2base(nn.Module):
+# def __init__(self):
+# super().__init__()
+# self.model = Wav2Vec2Model.from_pretrained("/data/docker/liujing04/gpt-vits/chinese-wav2vec2-base")
+# self.feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained("/data/docker/liujing04/gpt-vits/chinese-wav2vec2-base")
+# def forward(self, x):
+# input_values = self.feature_extractor(x, return_tensors="pt", sampling_rate=16000).input_values.to(x.device)
+# feats = self.model(input_values)["last_hidden_state"]
+# return feats
+
+
+
+def get_model():
+ model = CNHubert()
+ model.eval()
+ return model
+
+# def get_large_model():
+# model = CNHubertLarge()
+# model.eval()
+# return model
+#
+# def get_model_cvec():
+# model = CVec()
+# model.eval()
+# return model
+#
+# def get_model_cnw2v2base():
+# model = cnw2v2base()
+# model.eval()
+# return model
+
+def get_content(hmodel, wav_16k_tensor):
+ with torch.no_grad():
+ feats = hmodel(wav_16k_tensor)
+ return feats.transpose(1,2)
+
+
+if __name__ == '__main__':
+ model = get_model()
+ src_path = "/Users/Shared/原音频2.wav"
+ wav_16k_tensor = utils.load_wav_to_torch_and_resample(src_path, 16000)
+ model = model
+ wav_16k_tensor = wav_16k_tensor
+ feats = get_content(model,wav_16k_tensor)
+ print(feats.shape)
+
diff --git a/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/feature_extractor/whisper_enc.py b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/feature_extractor/whisper_enc.py
new file mode 100644
index 0000000000000000000000000000000000000000..023f751275bbf9d498b76a4da7ccd302e437949f
--- /dev/null
+++ b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/feature_extractor/whisper_enc.py
@@ -0,0 +1,22 @@
+import torch
+
+
+def get_model():
+ import whisper
+ model = whisper.load_model("small", device='cpu')
+
+ return model.encoder
+
+
+def get_content(model=None, wav_16k_tensor=None):
+ from whisper import log_mel_spectrogram, pad_or_trim
+ dev = next(model.parameters()).device
+ mel = log_mel_spectrogram(wav_16k_tensor).to(dev)[:, :3000]
+ # if torch.cuda.is_available():
+ # mel = mel.to(torch.float16)
+ feature_len = mel.shape[-1] // 2
+ assert mel.shape[-1] < 3000, "输入音频过长,只允许输入30以内音频"
+ with torch.no_grad():
+ feature = model(pad_or_trim(mel, 3000).unsqueeze(0))[:1, :feature_len, :].transpose(1,2)
+ return feature
+
diff --git a/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/inference_webui.py b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/inference_webui.py
new file mode 100644
index 0000000000000000000000000000000000000000..87e04fdbbc7e00b87717695e980746ec15940c88
--- /dev/null
+++ b/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/inference_webui.py
@@ -0,0 +1,270 @@
+import os
+gpt_path=os.environ.get("gpt_path","pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt")
+sovits_path=os.environ.get("sovits_path","pretrained_models/s2G488k.pth")
+cnhubert_base_path=os.environ.get("cnhubert_base_path","pretrained_models/chinese-hubert-base")
+bert_path=os.environ.get("bert_path","pretrained_models/chinese-roberta-wwm-ext-large")
+if("_CUDA_VISIBLE_DEVICES"in os.environ):
+ os.environ["CUDA_VISIBLE_DEVICES"]=os.environ["_CUDA_VISIBLE_DEVICES"]
+is_half=eval(os.environ.get("is_half","True"))
+import gradio as gr
+from transformers import AutoModelForMaskedLM, AutoTokenizer
+import sys,torch,numpy as np
+from pathlib import Path
+import os,pdb,utils,librosa,math,traceback,requests,argparse,torch,multiprocessing,pandas as pd,torch.multiprocessing as mp,soundfile
+# torch.backends.cuda.sdp_kernel("flash")
+# torch.backends.cuda.enable_flash_sdp(True)
+# torch.backends.cuda.enable_mem_efficient_sdp(True) # Not avaliable if torch version is lower than 2.0
+# torch.backends.cuda.enable_math_sdp(True)
+from random import shuffle
+from AR.utils import get_newest_ckpt
+from glob import glob
+from tqdm import tqdm
+from feature_extractor import cnhubert
+cnhubert.cnhubert_base_path=cnhubert_base_path
+from io import BytesIO
+from module.models import SynthesizerTrn
+from AR.models.t2s_lightning_module import Text2SemanticLightningModule
+from AR.utils.io import load_yaml_config
+from text import cleaned_text_to_sequence
+from text.cleaner import text_to_sequence, clean_text
+from time import time as ttime
+from module.mel_processing import spectrogram_torch
+from my_utils import load_audio
+
+device="cuda"
+tokenizer = AutoTokenizer.from_pretrained(bert_path)
+bert_model=AutoModelForMaskedLM.from_pretrained(bert_path)
+if(is_half==True):bert_model=bert_model.half().to(device)
+else:bert_model=bert_model.to(device)
+# bert_model=bert_model.to(device)
+def get_bert_feature(text, word2ph):
+ with torch.no_grad():
+ inputs = tokenizer(text, return_tensors="pt")
+ for i in inputs:
+ inputs[i] = inputs[i].to(device)#####输入是long不用管精度问题,精度随bert_model
+ res = bert_model(**inputs, output_hidden_states=True)
+ res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()[1:-1]
+ assert len(word2ph) == len(text)
+ phone_level_feature = []
+ for i in range(len(word2ph)):
+ repeat_feature = res[i].repeat(word2ph[i], 1)
+ phone_level_feature.append(repeat_feature)
+ phone_level_feature = torch.cat(phone_level_feature, dim=0)
+ # if(is_half==True):phone_level_feature=phone_level_feature.half()
+ return phone_level_feature.T
+
+n_semantic = 1024
+dict_s2=torch.load(sovits_path,map_location="cpu")
+hps=dict_s2["config"]
+class DictToAttrRecursive:
+ def __init__(self, input_dict):
+ for key, value in input_dict.items():
+ if isinstance(value, dict):
+ # 如果值是字典,递归调用构造函数
+ setattr(self, key, DictToAttrRecursive(value))
+ else:
+ setattr(self, key, value)
+
+hps = DictToAttrRecursive(hps)
+hps.model.semantic_frame_rate="25hz"
+dict_s1=torch.load(gpt_path,map_location="cpu")
+config=dict_s1["config"]
+ssl_model=cnhubert.get_model()
+if(is_half==True):ssl_model=ssl_model.half().to(device)
+else:ssl_model=ssl_model.to(device)
+
+vq_model = SynthesizerTrn(
+ hps.data.filter_length // 2 + 1,
+ hps.train.segment_size // hps.data.hop_length,
+ n_speakers=hps.data.n_speakers,
+ **hps.model)
+if(is_half==True):vq_model=vq_model.half().to(device)
+else:vq_model=vq_model.to(device)
+vq_model.eval()
+print(vq_model.load_state_dict(dict_s2["weight"],strict=False))
+hz = 50
+max_sec = config['data']['max_sec']
+# t2s_model = Text2SemanticLightningModule.load_from_checkpoint(checkpoint_path=gpt_path, config=config, map_location="cpu")#########todo
+t2s_model = Text2SemanticLightningModule(config,"ojbk",is_train=False)
+t2s_model.load_state_dict(dict_s1["weight"])
+if(is_half==True):t2s_model=t2s_model.half()
+t2s_model=t2s_model.to(device)
+t2s_model.eval()
+total = sum([param.nelement() for param in t2s_model.parameters()])
+print("Number of parameter: %.2fM" % (total / 1e6))
+def get_spepc(hps, filename):
+ audio=load_audio(filename,int(hps.data.sampling_rate))
+ audio=torch.FloatTensor(audio)
+ audio_norm = audio
+ audio_norm = audio_norm.unsqueeze(0)
+ spec = spectrogram_torch(audio_norm, hps.data.filter_length,hps.data.sampling_rate, hps.data.hop_length, hps.data.win_length,center=False)
+ return spec
+
+dict_language={
+ "中文":"zh",
+ "英文":"en",
+ "日文":"ja"
+}
+def get_tts_wav(ref_wav_path,prompt_text,prompt_language,text,text_language):
+ t0 = ttime()
+ prompt_text=prompt_text.strip("\n")
+ prompt_language,text=prompt_language,text.strip("\n")
+ with torch.no_grad():
+ wav16k, sr = librosa.load(ref_wav_path, sr=16000) # 派蒙
+ wav16k = torch.from_numpy(wav16k)
+ if(is_half==True):wav16k=wav16k.half().to(device)
+ else:wav16k=wav16k.to(device)
+ ssl_content = ssl_model.model(wav16k.unsqueeze(0))["last_hidden_state"].transpose(1, 2)#.float()
+ codes = vq_model.extract_latent(ssl_content)
+ prompt_semantic = codes[0, 0]
+ t1 = ttime()
+ prompt_language=dict_language[prompt_language]
+ text_language=dict_language[text_language]
+ phones1, word2ph1, norm_text1 = clean_text(prompt_text, prompt_language)
+ phones1=cleaned_text_to_sequence(phones1)
+ texts=text.split("\n")
+ audio_opt = []
+ zero_wav=np.zeros(int(hps.data.sampling_rate*0.3),dtype=np.float16 if is_half==True else np.float32)
+ for text in texts:
+ phones2, word2ph2, norm_text2 = clean_text(text, text_language)
+ phones2 = cleaned_text_to_sequence(phones2)
+ if(prompt_language=="zh"):bert1 = get_bert_feature(norm_text1, word2ph1)
+ else:bert1 = torch.zeros((1024, len(phones1)),dtype=torch.float16 if is_half==True else torch.float32).to(device)
+ if(text_language=="zh"):bert2 = get_bert_feature(norm_text2, word2ph2)
+ else:bert2 = torch.zeros((1024, len(phones2))).to(bert1)
+ bert = torch.cat([bert1, bert2], 1)
+
+ all_phoneme_ids = torch.LongTensor(phones1+phones2).to(device).unsqueeze(0)
+ bert = bert.to(device).unsqueeze(0)
+ all_phoneme_len = torch.tensor([all_phoneme_ids.shape[-1]]).to(device)
+ prompt = prompt_semantic.unsqueeze(0).to(device)
+ t2 = ttime()
+ with torch.no_grad():
+ # pred_semantic = t2s_model.model.infer(
+ pred_semantic,idx = t2s_model.model.infer_panel(
+ all_phoneme_ids,
+ all_phoneme_len,
+ prompt,
+ bert,
+ # prompt_phone_len=ph_offset,
+ top_k=config['inference']['top_k'],
+ early_stop_num=hz * max_sec)
+ t3 = ttime()
+ # print(pred_semantic.shape,idx)
+ pred_semantic = pred_semantic[:,-idx:].unsqueeze(0) # .unsqueeze(0)#mq要多unsqueeze一次
+ refer = get_spepc(hps, ref_wav_path)#.to(device)
+ if(is_half==True):refer=refer.half().to(device)
+ else:refer=refer.to(device)
+ # audio = vq_model.decode(pred_semantic, all_phoneme_ids, refer).detach().cpu().numpy()[0, 0]
+ audio = vq_model.decode(pred_semantic, torch.LongTensor(phones2).to(device).unsqueeze(0), refer).detach().cpu().numpy()[0, 0]###试试重建不带上prompt部分
+ audio_opt.append(audio)
+ audio_opt.append(zero_wav)
+ t4 = ttime()
+ print("%.3f\t%.3f\t%.3f\t%.3f" % (t1 - t0, t2 - t1, t3 - t2, t4 - t3))
+ yield hps.data.sampling_rate,(np.concatenate(audio_opt,0)*32768).astype(np.int16)
+
+
+splits={",","。","?","!",",",".","?","!","~",":",":","—","…",}#不考虑省略号
+def split(todo_text):
+ todo_text = todo_text.replace("……", "。").replace("——", ",")
+ if (todo_text[-1] not in splits): todo_text += "。"
+ i_split_head = i_split_tail = 0
+ len_text = len(todo_text)
+ todo_texts = []
+ while (1):
+ if (i_split_head >= len_text): break # 结尾一定有标点,所以直接跳出即可,最后一段在上次已加入
+ if (todo_text[i_split_head] in splits):
+ i_split_head += 1
+ todo_texts.append(todo_text[i_split_tail:i_split_head])
+ i_split_tail = i_split_head
+ else:
+ i_split_head += 1
+ return todo_texts
+def cut1(inp):
+ inp=inp.strip("\n")
+ inps=split(inp)
+ split_idx=list(range(0,len(inps),5))
+ split_idx[-1]=None
+ if(len(split_idx)>1):
+ opts=[]
+ for idx in range(len(split_idx)-1):
+ opts.append("".join(inps[split_idx[idx]:split_idx[idx+1]]))
+ else:
+ opts=[inp]
+ return "\n".join(opts)
+
+def cut2(inp):
+ inp=inp.strip("\n")
+ inps=split(inp)
+ if(len(inps)<2):return [inp]
+ opts=[]
+ summ=0
+ tmp_str=""
+ for i in range(len(inps)):
+ summ+=len(inps[i])
+ tmp_str+=inps[i]
+ if(summ>50):
+ summ=0
+ opts.append(tmp_str)
+ tmp_str=""
+ if(tmp_str!=""):opts.append(tmp_str)
+ if(len(opts[-1])<50):##如果最后一个太短了,和前一个合一起
+ opts[-2]=opts[-2]+opts[-1]
+ opts=opts[:-1]
+ return "\n".join(opts)
+
+def cut3(inp):
+ inp=inp.strip("\n")
+ return "\n".join(["%s。"%item for item in inp.strip("。").split("。")])
+
+with gr.Blocks(title="GPT-SoVITS WebUI") as app:
+ gr.Markdown(
+ value=
+ "本软件以MIT协议开源, 作者不对软件具备任何控制力, 使用软件者、传播软件导出的声音者自负全责.
+ for p in group["params"]:
+ state = self.state[p]
+ ...
+
+ you can do:
+
+ with self.batched_params(group["params"]) as batches:
+ for p, state, p_names in batches:
+ ...
+
+
+ Args:
+ group: a parameter group, which is a list of parameters; should be
+ one of self.param_groups.
+ group_params_names: name for each parameter in group,
+ which is List[str].
+ """
+ batches = defaultdict(
+ list
+ ) # `batches` maps from tuple (dtype_as_str,*shape) to list of nn.Parameter
+ batches_names = defaultdict(
+ list
+ ) # `batches` maps from tuple (dtype_as_str,*shape) to list of str
+
+ assert len(param_group) == len(group_params_names)
+ for p, named_p in zip(param_group, group_params_names):
+ key = (str(p.dtype), *p.shape)
+ batches[key].append(p)
+ batches_names[key].append(named_p)
+
+ batches_names_keys = list(batches_names.keys())
+ sorted_idx = sorted(
+ range(len(batches_names)), key=lambda i: batches_names_keys[i])
+ batches_names = [
+ batches_names[batches_names_keys[idx]] for idx in sorted_idx
+ ]
+ batches = [batches[batches_names_keys[idx]] for idx in sorted_idx]
+
+ stacked_params_dict = dict()
+
+ # turn batches into a list, in deterministic order.
+ # tuples will contain tuples of (stacked_param, state, stacked_params_names),
+ # one for each batch in `batches`.
+ tuples = []
+
+ for batch, batch_names in zip(batches, batches_names):
+ p = batch[0]
+ # we arbitrarily store the state in the
+ # state corresponding to the 1st parameter in the
+ # group. class Optimizer will take care of saving/loading state.
+ state = self.state[p]
+ p_stacked = torch.stack(batch)
+ grad = torch.stack([
+ torch.zeros_like(p) if p.grad is None else p.grad for p in batch
+ ])
+ p_stacked.grad = grad
+ stacked_params_dict[key] = p_stacked
+ tuples.append((p_stacked, state, batch_names))
+
+ yield tuples # <-- calling code will do the actual optimization here!
+
+ for ((stacked_params, _state, _names), batch) in zip(tuples, batches):
+ for i, p in enumerate(batch): # batch is list of Parameter
+ p.copy_(stacked_params[i])
+
+
+class ScaledAdam(BatchedOptimizer):
+ """
+ Implements 'Scaled Adam', a variant of Adam where we scale each parameter's update
+ proportional to the norm of that parameter; and also learn the scale of the parameter,
+ in log space, subject to upper and lower limits (as if we had factored each parameter as
+ param = underlying_param * log_scale.exp())
+
+
+ Args:
+ params: The parameters or param_groups to optimize (like other Optimizer subclasses)
+ lr: The learning rate. We will typically use a learning rate schedule that starts
+ at 0.03 and decreases over time, i.e. much higher than other common
+ optimizers.
+ clipping_scale: (e.g. 2.0)
+ A scale for gradient-clipping: if specified, the normalized gradients
+ over the whole model will be clipped to have 2-norm equal to
+ `clipping_scale` times the median 2-norm over the most recent period
+ of `clipping_update_period` minibatches. By "normalized gradients",
+ we mean after multiplying by the rms parameter value for this tensor
+ [for non-scalars]; this is appropriate because our update is scaled
+ by this quantity.
+ betas: beta1,beta2 are momentum constants for regular momentum, and moving sum-sq grad.
+ Must satisfy 0 < beta <= beta2 < 1.
+ scalar_lr_scale: A scaling factor on the learning rate, that we use to update the
+ scale of each parameter tensor and scalar parameters of the mode..
+ If each parameter were decomposed
+ as p * p_scale.exp(), where (p**2).mean().sqrt() == 1.0, scalar_lr_scale
+ would be a the scaling factor on the learning rate of p_scale.
+ eps: A general-purpose epsilon to prevent division by zero
+ param_min_rms: Minimum root-mean-square value of parameter tensor, for purposes of
+ learning the scale on the parameters (we'll constrain the rms of each non-scalar
+ parameter tensor to be >= this value)
+ param_max_rms: Maximum root-mean-square value of parameter tensor, for purposes of
+ learning the scale on the parameters (we'll constrain the rms of each non-scalar
+ parameter tensor to be <= this value)
+ scalar_max: Maximum absolute value for scalar parameters (applicable if your
+ model has any parameters with numel() == 1).
+ size_update_period: The periodicity, in steps, with which we update the size (scale)
+ of the parameter tensor. This is provided to save a little time
+ in the update.
+ clipping_update_period: if clipping_scale is specified, this is the period
+ """
+
+ def __init__(
+ self,
+ params,
+ lr=3e-02,
+ clipping_scale=None,
+ betas=(0.9, 0.98),
+ scalar_lr_scale=0.1,
+ eps=1.0e-08,
+ param_min_rms=1.0e-05,
+ param_max_rms=3.0,
+ scalar_max=10.0,
+ size_update_period=4,
+ clipping_update_period=100,
+ parameters_names=None,
+ show_dominant_parameters=True, ):
+
+ assert parameters_names is not None, (
+ "Please prepare parameters_names,"
+ "which is a List[List[str]]. Each List[str] is for a group"
+ "and each str is for a parameter")
+ defaults = dict(
+ lr=lr,
+ clipping_scale=clipping_scale,
+ betas=betas,
+ scalar_lr_scale=scalar_lr_scale,
+ eps=eps,
+ param_min_rms=param_min_rms,
+ param_max_rms=param_max_rms,
+ scalar_max=scalar_max,
+ size_update_period=size_update_period,
+ clipping_update_period=clipping_update_period, )
+
+ super(ScaledAdam, self).__init__(params, defaults)
+ assert len(self.param_groups) == len(parameters_names)
+ self.parameters_names = parameters_names
+ self.show_dominant_parameters = show_dominant_parameters
+
+ def __setstate__(self, state):
+ super(ScaledAdam, self).__setstate__(state)
+
+ @torch.no_grad()
+ def step(self, closure=None):
+ """Performs a single optimization step.
+
+ Arguments:
+ closure (callable, optional): A closure that reevaluates the model
+ and returns the loss.
+ """
+ loss = None
+ if closure is not None:
+ with torch.enable_grad():
+ loss = closure()
+
+ batch = True
+
+ for group, group_params_names in zip(self.param_groups,
+ self.parameters_names):
+
+ with self.batched_params(group["params"],
+ group_params_names) as batches:
+
+ # batches is list of pairs (stacked_param, state). stacked_param is like
+ # a regular parameter, and will have a .grad, but the 1st dim corresponds to
+ # a stacking dim, it is not a real dim.
+
+ if (len(batches[0][1]) ==
+ 0): # if len(first state) == 0: not yet initialized
+ clipping_scale = 1
+ else:
+ clipping_scale = self._get_clipping_scale(group, batches)
+
+ for p, state, _ in batches:
+ # Perform optimization step.
+ # grad is not going to be None, we handled that when creating the batches.
+ grad = p.grad
+ if grad.is_sparse:
+ raise RuntimeError(
+ "ScaledAdam optimizer does not support sparse gradients"
+ )
+ # State initialization
+ if len(state) == 0:
+ self._init_state(group, p, state)
+
+ self._step_one_batch(group, p, state, clipping_scale)
+
+ return loss
+
+ def _init_state(self, group: dict, p: Tensor, state: dict):
+ """
+ Initializes state dict for parameter 'p'. Assumes that dim 0 of tensor p
+ is actually the batch dimension, corresponding to batched-together
+ parameters of a given shape.
+
+
+ Args:
+ group: Dict to look up configuration values.
+ p: The parameter that we are initializing the state for
+ state: Dict from string to whatever state we are initializing
+ """
+ size_update_period = group["size_update_period"]
+
+ state["step"] = 0
+
+ kwargs = {"device": p.device, "dtype": p.dtype}
+
+ # 'delta' implements conventional momentum. There are
+ # several different kinds of update going on, so rather than
+ # compute "exp_avg" like in Adam, we store and decay a
+ # parameter-change "delta", which combines all forms of
+ # update. this is equivalent to how it's done in Adam,
+ # except for the first few steps.
+ state["delta"] = torch.zeros_like(
+ p, memory_format=torch.preserve_format)
+
+ batch_size = p.shape[0]
+ numel = p.numel() // batch_size
+ numel = p.numel()
+
+ if numel > 1:
+ # "param_rms" just periodically records the scalar root-mean-square value of
+ # the parameter tensor.
+ # it has a shape like (batch_size, 1, 1, 1, 1)
+ param_rms = (
+ (p**2).mean(dim=list(range(1, p.ndim)), keepdim=True).sqrt())
+ state["param_rms"] = param_rms
+
+ state["scale_exp_avg_sq"] = torch.zeros_like(param_rms)
+ state["scale_grads"] = torch.zeros(size_update_period,
+ *param_rms.shape, **kwargs)
+
+ # exp_avg_sq is the weighted sum of scaled gradients. as in Adam.
+ state["exp_avg_sq"] = torch.zeros_like(
+ p, memory_format=torch.preserve_format)
+
+ def _get_clipping_scale(self,
+ group: dict,
+ tuples: List[Tuple[Tensor, dict, List[str]]]
+ ) -> float:
+ """
+ Returns a scalar factor <= 1.0 that dictates gradient clipping, i.e. we will scale the gradients
+ by this amount before applying the rest of the update.
+
+ Args:
+ group: the parameter group, an item in self.param_groups
+ tuples: a list of tuples of (param, state, param_names)
+ where param is a batched set of parameters,
+ with a .grad (1st dim is batch dim)
+ and state is the state-dict where optimization parameters are kept.
+ param_names is a List[str] while each str is name for a parameter
+ in batched set of parameters "param".
+ """
+ assert len(tuples) >= 1
+ clipping_scale = group["clipping_scale"]
+ (first_p, first_state, _) = tuples[0]
+ step = first_state["step"]
+ if clipping_scale is None or step == 0:
+ # no clipping. return early on step == 0 because the other
+ # parameters' state won't have been initialized yet.
+ return 1.0
+ clipping_update_period = group["clipping_update_period"]
+
+ tot_sumsq = torch.tensor(0.0, device=first_p.device)
+ for (p, state, param_names) in tuples:
+ grad = p.grad
+ if grad.is_sparse:
+ raise RuntimeError(
+ "ScaledAdam optimizer does not support sparse gradients")
+ if p.numel() == p.shape[0]: # a batch of scalars
+ tot_sumsq += (grad**2).sum() # sum() to change shape [1] to []
+ else:
+ tot_sumsq += ((grad * state["param_rms"])**2).sum()
+
+ tot_norm = tot_sumsq.sqrt()
+ if "model_norms" not in first_state:
+ first_state["model_norms"] = torch.zeros(
+ clipping_update_period, device=p.device)
+ first_state["model_norms"][step % clipping_update_period] = tot_norm
+
+ if step % clipping_update_period == 0:
+ # Print some stats.
+ # We don't reach here if step == 0 because we would have returned
+ # above.
+ sorted_norms = first_state["model_norms"].sort()[0].to("cpu")
+ quartiles = []
+ for n in range(0, 5):
+ index = min(
+ clipping_update_period - 1,
+ (clipping_update_period // 4) * n, )
+ quartiles.append(sorted_norms[index].item())
+
+ median = quartiles[2]
+ threshold = clipping_scale * median
+ first_state["model_norm_threshold"] = threshold
+ percent_clipped = (first_state["num_clipped"] * 100.0 /
+ clipping_update_period
+ if "num_clipped" in first_state else 0.0)
+ first_state["num_clipped"] = 0
+ quartiles = " ".join(["%.3e" % x for x in quartiles])
+ logging.info(
+ f"Clipping_scale={clipping_scale}, grad-norm quartiles {quartiles}, "
+ f"threshold={threshold:.3e}, percent-clipped={percent_clipped:.1f}"
+ )
+
+ if step < clipping_update_period:
+ return 1.0 # We have not yet estimated a norm to clip to.
+ else:
+ try:
+ model_norm_threshold = first_state["model_norm_threshold"]
+ except KeyError:
+ logging.info(
+ "Warning: model_norm_threshold not in state: possibly "
+ "you changed config when restarting, adding clipping_scale option?"
+ )
+ return 1.0
+ ans = min(1.0, (model_norm_threshold / (tot_norm + 1.0e-20)).item())
+ if ans < 1.0:
+ first_state["num_clipped"] += 1
+ if ans < 0.1:
+ logging.warn(
+ f"Scaling gradients by {ans}, model_norm_threshold={model_norm_threshold}"
+ )
+ if self.show_dominant_parameters:
+ assert p.shape[0] == len(param_names)
+ self._show_gradient_dominating_parameter(tuples, tot_sumsq)
+ return ans
+
+ def _show_gradient_dominating_parameter(
+ self, tuples: List[Tuple[Tensor, dict, List[str]]],
+ tot_sumsq: Tensor):
+ """
+ Show information of parameter wihch dominanting tot_sumsq.
+
+ Args:
+ tuples: a list of tuples of (param, state, param_names)
+ where param is a batched set of parameters,
+ with a .grad (1st dim is batch dim)
+ and state is the state-dict where optimization parameters are kept.
+ param_names is a List[str] while each str is name for a parameter
+ in batched set of parameters "param".
+ tot_sumsq: sumsq of all parameters. Though it's could be calculated
+ from tuples, we still pass it to save some time.
+ """
+ all_sumsq_orig = {}
+ for (p, state, batch_param_names) in tuples:
+ # p is a stacked batch parameters.
+ batch_grad = p.grad
+ if p.numel() == p.shape[0]: # a batch of scalars
+ batch_sumsq_orig = batch_grad**2
+ # Dummpy values used by following `zip` statement.
+ batch_rms_orig = torch.ones(p.shape[0])
+ else:
+ batch_rms_orig = state["param_rms"]
+ batch_sumsq_orig = ((batch_grad * batch_rms_orig)**2).sum(
+ dim=list(range(1, batch_grad.ndim)))
+
+ for name, sumsq_orig, rms, grad in zip(batch_param_names,
+ batch_sumsq_orig,
+ batch_rms_orig, batch_grad):
+
+ proportion_orig = sumsq_orig / tot_sumsq
+ all_sumsq_orig[name] = (proportion_orig, sumsq_orig, rms, grad)
+
+ assert torch.isclose(
+ sum([value[0] for value in all_sumsq_orig.values()]).cpu(),
+ torch.tensor(1.0), )
+ sorted_by_proportion = {
+ k: v
+ for k, v in sorted(
+ all_sumsq_orig.items(),
+ key=lambda item: item[1][0],
+ reverse=True, )
+ }
+ dominant_param_name = next(iter(sorted_by_proportion))
+ (dominant_proportion, dominant_sumsq, dominant_rms,
+ dominant_grad, ) = sorted_by_proportion[dominant_param_name]
+ logging.info(f"Parameter Dominanting tot_sumsq {dominant_param_name}"
+ f" with proportion {dominant_proportion:.2f},"
+ f" where dominant_sumsq=(grad_sumsq*orig_rms_sq)"
+ f"={dominant_sumsq:.3e},"
+ f" grad_sumsq = {(dominant_grad**2).sum():.3e},"
+ f" orig_rms_sq={(dominant_rms**2).item():.3e}")
+
+ def _step_one_batch(self,
+ group: dict,
+ p: Tensor,
+ state: dict,
+ clipping_scale: float):
+ """
+ Do the step for one parameter, which is actually going to be a batch of
+ `real` parameters, with dim 0 as the batch dim.
+ Args:
+ group: dict to look up configuration values
+ p: parameter to update (actually multiple parameters stacked together
+ as a batch)
+ state: state-dict for p, to look up the optimizer state
+ """
+ lr = group["lr"]
+ size_update_period = group["size_update_period"]
+ beta1 = group["betas"][0]
+
+ grad = p.grad
+ if clipping_scale != 1.0:
+ grad = grad * clipping_scale
+ step = state["step"]
+ delta = state["delta"]
+
+ delta.mul_(beta1)
+ batch_size = p.shape[0]
+ numel = p.numel() // batch_size
+ if numel > 1:
+ # Update the size/scale of p, and set param_rms
+ scale_grads = state["scale_grads"]
+ scale_grads[step % size_update_period] = (p * grad).sum(
+ dim=list(range(1, p.ndim)), keepdim=True)
+ if step % size_update_period == size_update_period - 1:
+ param_rms = state["param_rms"] # shape: (batch_size, 1, 1, ..)
+ param_rms.copy_((p**2)
+ .mean(dim=list(range(1, p.ndim)), keepdim=True)
+ .sqrt())
+ if step > 0:
+ # self._size_update() learns the overall scale on the
+ # parameter, by shrinking or expanding it.
+ self._size_update(group, scale_grads, p, state)
+
+ if numel == 1:
+ # For parameters with 1 element we just use regular Adam.
+ # Updates delta.
+ self._step_scalar(group, p, state)
+ else:
+ self._step(group, p, state)
+
+ state["step"] = step + 1
+
+ def _size_update(self,
+ group: dict,
+ scale_grads: Tensor,
+ p: Tensor,
+ state: dict) -> None:
+ """
+ Called only where p.numel() > 1, this updates the scale of the parameter.
+ If we imagine: p = underlying_param * scale.exp(), and we are doing
+ gradient descent on underlying param and on scale, this function does the update
+ on `scale`.
+
+ Args:
+ group: dict to look up configuration values
+ scale_grads: a tensor of shape (size_update_period, batch_size, 1, 1,...) containing
+ grads w.r.t. the scales.
+ p: The parameter to update
+ state: The state-dict of p
+ """
+
+ param_rms = state["param_rms"]
+ beta1, beta2 = group["betas"]
+ size_lr = group["lr"] * group["scalar_lr_scale"]
+ param_min_rms = group["param_min_rms"]
+ param_max_rms = group["param_max_rms"]
+ eps = group["eps"]
+ step = state["step"]
+ batch_size = p.shape[0]
+
+ size_update_period = scale_grads.shape[0]
+ # correct beta2 for the size update period: we will have
+ # faster decay at this level.
+ beta2_corr = beta2**size_update_period
+
+ scale_exp_avg_sq = state[
+ "scale_exp_avg_sq"] # shape: (batch_size, 1, 1, ..)
+ scale_exp_avg_sq.mul_(beta2_corr).add_(
+ (scale_grads**2).mean(dim=0), # mean over dim `size_update_period`
+ alpha=1 - beta2_corr, ) # shape is (batch_size, 1, 1, ...)
+
+ # The 1st time we reach here is when size_step == 1.
+ size_step = (step + 1) // size_update_period
+ bias_correction2 = 1 - beta2_corr**size_step
+ # we don't bother with bias_correction1; this will help prevent divergence
+ # at the start of training.
+
+ denom = scale_exp_avg_sq.sqrt() + eps
+
+ scale_step = (-size_lr * (bias_correction2**0.5) *
+ scale_grads.sum(dim=0) / denom)
+
+ is_too_small = param_rms < param_min_rms
+ is_too_large = param_rms > param_max_rms
+
+ # when the param gets too small, just don't shrink it any further.
+ scale_step.masked_fill_(is_too_small, 0.0)
+ # when it gets too large, stop it from getting any larger.
+ scale_step.masked_fill_(is_too_large, -size_lr * size_update_period)
+ delta = state["delta"]
+ # the factor of (1-beta1) relates to momentum.
+ delta.add_(p * scale_step, alpha=(1 - beta1))
+
+ def _step(self, group: dict, p: Tensor, state: dict):
+ """
+ This function does the core update of self.step(), in the case where the members of
+ the batch have more than 1 element.
+
+ Args:
+ group: A dict which will be used to look up configuration values
+ p: The parameter to be updated
+ grad: The grad of p
+ state: The state-dict corresponding to parameter p
+
+ This function modifies p.
+ """
+ grad = p.grad
+ lr = group["lr"]
+ beta1, beta2 = group["betas"]
+ eps = group["eps"]
+ param_min_rms = group["param_min_rms"]
+ step = state["step"]
+
+ exp_avg_sq = state["exp_avg_sq"]
+ exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=(1 - beta2))
+
+ this_step = state["step"] - (state["zero_step"]
+ if "zero_step" in state else 0)
+ bias_correction2 = 1 - beta2**(this_step + 1)
+ if bias_correction2 < 0.99:
+ # note: not in-place.
+ exp_avg_sq = exp_avg_sq * (1.0 / bias_correction2)
+
+ denom = exp_avg_sq.sqrt()
+ denom += eps
+ grad = grad / denom
+
+ alpha = -lr * (1 - beta1) * state["param_rms"].clamp(min=param_min_rms)
+
+ delta = state["delta"]
+ delta.add_(grad * alpha)
+ p.add_(delta)
+
+ def _step_scalar(self, group: dict, p: Tensor, state: dict):
+ """
+ A simplified form of the core update for scalar tensors, where we cannot get a good
+ estimate of the parameter rms.
+ """
+ beta1, beta2 = group["betas"]
+ scalar_max = group["scalar_max"]
+ eps = group["eps"]
+ lr = group["lr"] * group["scalar_lr_scale"]
+ grad = p.grad
+
+ exp_avg_sq = state["exp_avg_sq"] # shape: (batch_size,)
+ exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)
+
+ # bias_correction2 is like in Adam. Don't bother with bias_correction1;
+ # slower update at the start will help stability anyway.
+ bias_correction2 = 1 - beta2**(state["step"] + 1)
+ denom = (exp_avg_sq / bias_correction2).sqrt() + eps
+
+ delta = state["delta"]
+ delta.add_(grad / denom, alpha=-lr * (1 - beta1))
+ p.clamp_(min=-scalar_max, max=scalar_max)
+ p.add_(delta)
diff --git a/GPT_SoVITS/AR/modules/patched_mha_with_cache.py b/GPT_SoVITS/AR/modules/patched_mha_with_cache.py
new file mode 100644
index 0000000000000000000000000000000000000000..7be241dadd378fc9312916f60433ba4b7aa7c764
--- /dev/null
+++ b/GPT_SoVITS/AR/modules/patched_mha_with_cache.py
@@ -0,0 +1,465 @@
+from torch.nn.functional import *
+from torch.nn.functional import (
+ _mha_shape_check,
+ _canonical_mask,
+ _none_or_dtype,
+ _in_projection_packed,
+)
+from torch.nn import functional as F
+import torch
+# Tensor = torch.Tensor
+# from typing import Callable, List, Optional, Tuple, Union
+
+
+def multi_head_attention_forward_patched(
+ query: Tensor,
+ key: Tensor,
+ value: Tensor,
+ embed_dim_to_check: int,
+ num_heads: int,
+ in_proj_weight: Optional[Tensor],
+ in_proj_bias: Optional[Tensor],
+ bias_k: Optional[Tensor],
+ bias_v: Optional[Tensor],
+ add_zero_attn: bool,
+ dropout_p: float,
+ out_proj_weight: Tensor,
+ out_proj_bias: Optional[Tensor],
+ training: bool = True,
+ key_padding_mask: Optional[Tensor] = None,
+ need_weights: bool = True,
+ attn_mask: Optional[Tensor] = None,
+ use_separate_proj_weight: bool = False,
+ q_proj_weight: Optional[Tensor] = None,
+ k_proj_weight: Optional[Tensor] = None,
+ v_proj_weight: Optional[Tensor] = None,
+ static_k: Optional[Tensor] = None,
+ static_v: Optional[Tensor] = None,
+ average_attn_weights: bool = True,
+ is_causal: bool = False,
+ cache=None,
+) -> Tuple[Tensor, Optional[Tensor]]:
+ r"""
+ Args:
+ query, key, value: map a query and a set of key-value pairs to an output.
+ See "Attention Is All You Need" for more details.
+ embed_dim_to_check: total dimension of the model.
+ num_heads: parallel attention heads.
+ in_proj_weight, in_proj_bias: input projection weight and bias.
+ bias_k, bias_v: bias of the key and value sequences to be added at dim=0.
+ add_zero_attn: add a new batch of zeros to the key and
+ value sequences at dim=1.
+ dropout_p: probability of an element to be zeroed.
+ out_proj_weight, out_proj_bias: the output projection weight and bias.
+ training: apply dropout if is ``True``.
+ key_padding_mask: if provided, specified padding elements in the key will
+ be ignored by the attention. This is an binary mask. When the value is True,
+ the corresponding value on the attention layer will be filled with -inf.
+ need_weights: output attn_output_weights.
+ Default: `True`
+ Note: `needs_weight` defaults to `True`, but should be set to `False`
+ For best performance when attention weights are not nedeeded.
+ *Setting needs_weights to `True`
+ leads to a significant performance degradation.*
+ attn_mask: 2D or 3D mask that prevents attention to certain positions. A 2D mask will be broadcasted for all
+ the batches while a 3D mask allows to specify a different mask for the entries of each batch.
+ is_causal: If specified, applies a causal mask as attention mask, and ignores
+ attn_mask for computing scaled dot product attention.
+ Default: ``False``.
+ .. warning::
+ is_causal is provides a hint that the attn_mask is the
+ causal mask.Providing incorrect hints can result in
+ incorrect execution, including forward and backward
+ compatibility.
+ use_separate_proj_weight: the function accept the proj. weights for query, key,
+ and value in different forms. If false, in_proj_weight will be used, which is
+ a combination of q_proj_weight, k_proj_weight, v_proj_weight.
+ q_proj_weight, k_proj_weight, v_proj_weight, in_proj_bias: input projection weight and bias.
+ static_k, static_v: static key and value used for attention operators.
+ average_attn_weights: If true, indicates that the returned ``attn_weights`` should be averaged across heads.
+ Otherwise, ``attn_weights`` are provided separately per head. Note that this flag only has an effect
+ when ``need_weights=True.``. Default: True
+
+
+ Shape:
+ Inputs:
+ - query: :math:`(L, E)` or :math:`(L, N, E)` where L is the target sequence length, N is the batch size, E is
+ the embedding dimension.
+ - key: :math:`(S, E)` or :math:`(S, N, E)`, where S is the source sequence length, N is the batch size, E is
+ the embedding dimension.
+ - value: :math:`(S, E)` or :math:`(S, N, E)` where S is the source sequence length, N is the batch size, E is
+ the embedding dimension.
+ - key_padding_mask: :math:`(S)` or :math:`(N, S)` where N is the batch size, S is the source sequence length.
+ If a FloatTensor is provided, it will be directly added to the value.
+ If a BoolTensor is provided, the positions with the
+ value of ``True`` will be ignored while the position with the value of ``False`` will be unchanged.
+ - attn_mask: 2D mask :math:`(L, S)` where L is the target sequence length, S is the source sequence length.
+ 3D mask :math:`(N*num_heads, L, S)` where N is the batch size, L is the target sequence length,
+ S is the source sequence length. attn_mask ensures that position i is allowed to attend the unmasked
+ positions. If a BoolTensor is provided, positions with ``True``
+ are not allowed to attend while ``False`` values will be unchanged. If a FloatTensor
+ is provided, it will be added to the attention weight.
+ - static_k: :math:`(N*num_heads, S, E/num_heads)`, where S is the source sequence length,
+ N is the batch size, E is the embedding dimension. E/num_heads is the head dimension.
+ - static_v: :math:`(N*num_heads, S, E/num_heads)`, where S is the source sequence length,
+ N is the batch size, E is the embedding dimension. E/num_heads is the head dimension.
+
+ Outputs:
+ - attn_output: :math:`(L, E)` or :math:`(L, N, E)` where L is the target sequence length, N is the batch size,
+ E is the embedding dimension.
+ - attn_output_weights: Only returned when ``need_weights=True``. If ``average_attn_weights=True``, returns
+ attention weights averaged across heads of shape :math:`(L, S)` when input is unbatched or
+ :math:`(N, L, S)`, where :math:`N` is the batch size, :math:`L` is the target sequence length, and
+ :math:`S` is the source sequence length. If ``average_attn_weights=False``, returns attention weights per
+ head of shape :math:`(num_heads, L, S)` when input is unbatched or :math:`(N, num_heads, L, S)`.
+ """
+ tens_ops = (
+ query,
+ key,
+ value,
+ in_proj_weight,
+ in_proj_bias,
+ bias_k,
+ bias_v,
+ out_proj_weight,
+ out_proj_bias,
+ )
+ if has_torch_function(tens_ops):
+ return handle_torch_function(
+ multi_head_attention_forward,
+ tens_ops,
+ query,
+ key,
+ value,
+ embed_dim_to_check,
+ num_heads,
+ in_proj_weight,
+ in_proj_bias,
+ bias_k,
+ bias_v,
+ add_zero_attn,
+ dropout_p,
+ out_proj_weight,
+ out_proj_bias,
+ training=training,
+ key_padding_mask=key_padding_mask,
+ need_weights=need_weights,
+ attn_mask=attn_mask,
+ is_causal=is_causal,
+ use_separate_proj_weight=use_separate_proj_weight,
+ q_proj_weight=q_proj_weight,
+ k_proj_weight=k_proj_weight,
+ v_proj_weight=v_proj_weight,
+ static_k=static_k,
+ static_v=static_v,
+ average_attn_weights=average_attn_weights,
+ cache=cache,
+ )
+
+ is_batched = _mha_shape_check(
+ query, key, value, key_padding_mask, attn_mask, num_heads
+ )
+
+ # For unbatched input, we unsqueeze at the expected batch-dim to pretend that the input
+ # is batched, run the computation and before returning squeeze the
+ # batch dimension so that the output doesn't carry this temporary batch dimension.
+ if not is_batched:
+ # unsqueeze if the input is unbatched
+ query = query.unsqueeze(1)
+ key = key.unsqueeze(1)
+ value = value.unsqueeze(1)
+ if key_padding_mask is not None:
+ key_padding_mask = key_padding_mask.unsqueeze(0)
+
+ # set up shape vars
+ tgt_len, bsz, embed_dim = query.shape
+ src_len, _, _ = key.shape
+
+ key_padding_mask = _canonical_mask(
+ mask=key_padding_mask,
+ mask_name="key_padding_mask",
+ other_type=_none_or_dtype(attn_mask),
+ other_name="attn_mask",
+ target_type=query.dtype,
+ )
+
+ if is_causal and attn_mask is None:
+ raise RuntimeError(
+ "Need attn_mask if specifying the is_causal hint. "
+ "You may use the Transformer module method "
+ "`generate_square_subsequent_mask` to create this mask."
+ )
+
+ if is_causal and key_padding_mask is None and not need_weights:
+ # when we have a kpm or need weights, we need attn_mask
+ # Otherwise, we use the is_causal hint go as is_causal
+ # indicator to SDPA.
+ attn_mask = None
+ else:
+ attn_mask = _canonical_mask(
+ mask=attn_mask,
+ mask_name="attn_mask",
+ other_type=None,
+ other_name="",
+ target_type=query.dtype,
+ check_other=False,
+ )
+
+ if key_padding_mask is not None:
+ # We have the attn_mask, and use that to merge kpm into it.
+ # Turn off use of is_causal hint, as the merged mask is no
+ # longer causal.
+ is_causal = False
+
+ assert (
+ embed_dim == embed_dim_to_check
+ ), f"was expecting embedding dimension of {embed_dim_to_check}, but got {embed_dim}"
+ if isinstance(embed_dim, torch.Tensor):
+ # embed_dim can be a tensor when JIT tracing
+ head_dim = embed_dim.div(num_heads, rounding_mode="trunc")
+ else:
+ head_dim = embed_dim // num_heads
+ assert (
+ head_dim * num_heads == embed_dim
+ ), f"embed_dim {embed_dim} not divisible by num_heads {num_heads}"
+ if use_separate_proj_weight:
+ # allow MHA to have different embedding dimensions when separate projection weights are used
+ assert (
+ key.shape[:2] == value.shape[:2]
+ ), f"key's sequence and batch dims {key.shape[:2]} do not match value's {value.shape[:2]}"
+ else:
+ assert (
+ key.shape == value.shape
+ ), f"key shape {key.shape} does not match value shape {value.shape}"
+
+ #
+ # compute in-projection
+ #
+ if not use_separate_proj_weight:
+ assert (
+ in_proj_weight is not None
+ ), "use_separate_proj_weight is False but in_proj_weight is None"
+ q, k, v = _in_projection_packed(query, key, value, in_proj_weight, in_proj_bias)
+ else:
+ assert (
+ q_proj_weight is not None
+ ), "use_separate_proj_weight is True but q_proj_weight is None"
+ assert (
+ k_proj_weight is not None
+ ), "use_separate_proj_weight is True but k_proj_weight is None"
+ assert (
+ v_proj_weight is not None
+ ), "use_separate_proj_weight is True but v_proj_weight is None"
+ if in_proj_bias is None:
+ b_q = b_k = b_v = None
+ else:
+ b_q, b_k, b_v = in_proj_bias.chunk(3)
+ q, k, v = _in_projection(
+ query,
+ key,
+ value,
+ q_proj_weight,
+ k_proj_weight,
+ v_proj_weight,
+ b_q,
+ b_k,
+ b_v,
+ )
+ if cache != None:
+ if cache["first_infer"] == 1:
+ cache["k"][cache["stage"]] = k
+ # print(0,cache["k"].shape)
+ cache["v"][cache["stage"]] = v
+ else: ###12个layer每个都要留自己的cache_kv
+ # print(1,cache["k"].shape)
+ cache["k"][cache["stage"]] = torch.cat(
+ [cache["k"][cache["stage"]], k], 0
+ ) ##本来时序是1,但是proj的时候可能transpose了所以时序到0维了
+ cache["v"][cache["stage"]] = torch.cat([cache["v"][cache["stage"]], v], 0)
+ # print(2, cache["k"].shape)
+ src_len = cache["k"][cache["stage"]].shape[0]
+ k = cache["k"][cache["stage"]]
+ v = cache["v"][cache["stage"]]
+ # if attn_mask is not None:
+ # attn_mask=attn_mask[-1:,]
+ # print(attn_mask.shape,attn_mask)
+ cache["stage"] = (cache["stage"] + 1) % cache["all_stage"]
+ # print(2333,cache)
+ # prep attention mask
+
+ attn_mask = _canonical_mask(
+ mask=attn_mask,
+ mask_name="attn_mask",
+ other_type=None,
+ other_name="",
+ target_type=q.dtype,
+ check_other=False,
+ )
+
+ if attn_mask is not None:
+ # ensure attn_mask's dim is 3
+ if attn_mask.dim() == 2:
+ correct_2d_size = (tgt_len, src_len)
+ if attn_mask.shape != correct_2d_size:
+ raise RuntimeError(
+ f"The shape of the 2D attn_mask is {attn_mask.shape}, but should be {correct_2d_size}."
+ )
+ attn_mask = attn_mask.unsqueeze(0)
+ elif attn_mask.dim() == 3:
+ correct_3d_size = (bsz * num_heads, tgt_len, src_len)
+ if attn_mask.shape != correct_3d_size:
+ raise RuntimeError(
+ f"The shape of the 3D attn_mask is {attn_mask.shape}, but should be {correct_3d_size}."
+ )
+ else:
+ raise RuntimeError(
+ f"attn_mask's dimension {attn_mask.dim()} is not supported"
+ )
+
+ # add bias along batch dimension (currently second)
+ if bias_k is not None and bias_v is not None:
+ assert static_k is None, "bias cannot be added to static key."
+ assert static_v is None, "bias cannot be added to static value."
+ k = torch.cat([k, bias_k.repeat(1, bsz, 1)])
+ v = torch.cat([v, bias_v.repeat(1, bsz, 1)])
+ if attn_mask is not None:
+ attn_mask = pad(attn_mask, (0, 1))
+ if key_padding_mask is not None:
+ key_padding_mask = pad(key_padding_mask, (0, 1))
+ else:
+ assert bias_k is None
+ assert bias_v is None
+
+ #
+ # reshape q, k, v for multihead attention and make em batch first
+ #
+ q = q.view(tgt_len, bsz * num_heads, head_dim).transpose(0, 1)
+ if static_k is None:
+ k = k.view(k.shape[0], bsz * num_heads, head_dim).transpose(0, 1)
+ else:
+ # TODO finish disentangling control flow so we don't do in-projections when statics are passed
+ assert (
+ static_k.size(0) == bsz * num_heads
+ ), f"expecting static_k.size(0) of {bsz * num_heads}, but got {static_k.size(0)}"
+ assert (
+ static_k.size(2) == head_dim
+ ), f"expecting static_k.size(2) of {head_dim}, but got {static_k.size(2)}"
+ k = static_k
+ if static_v is None:
+ v = v.view(v.shape[0], bsz * num_heads, head_dim).transpose(0, 1)
+ else:
+ # TODO finish disentangling control flow so we don't do in-projections when statics are passed
+ assert (
+ static_v.size(0) == bsz * num_heads
+ ), f"expecting static_v.size(0) of {bsz * num_heads}, but got {static_v.size(0)}"
+ assert (
+ static_v.size(2) == head_dim
+ ), f"expecting static_v.size(2) of {head_dim}, but got {static_v.size(2)}"
+ v = static_v
+
+ # add zero attention along batch dimension (now first)
+ if add_zero_attn:
+ zero_attn_shape = (bsz * num_heads, 1, head_dim)
+ k = torch.cat(
+ [k, torch.zeros(zero_attn_shape, dtype=k.dtype, device=k.device)], dim=1
+ )
+ v = torch.cat(
+ [v, torch.zeros(zero_attn_shape, dtype=v.dtype, device=v.device)], dim=1
+ )
+ if attn_mask is not None:
+ attn_mask = pad(attn_mask, (0, 1))
+ if key_padding_mask is not None:
+ key_padding_mask = pad(key_padding_mask, (0, 1))
+
+ # update source sequence length after adjustments
+ src_len = k.size(1)
+
+ # merge key padding and attention masks
+ if key_padding_mask is not None:
+ assert key_padding_mask.shape == (
+ bsz,
+ src_len,
+ ), f"expecting key_padding_mask shape of {(bsz, src_len)}, but got {key_padding_mask.shape}"
+ key_padding_mask = (
+ key_padding_mask.view(bsz, 1, 1, src_len)
+ .expand(-1, num_heads, -1, -1)
+ .reshape(bsz * num_heads, 1, src_len)
+ )
+ if attn_mask is None:
+ attn_mask = key_padding_mask
+ else:
+ attn_mask = attn_mask + key_padding_mask
+
+ # adjust dropout probability
+ if not training:
+ dropout_p = 0.0
+
+ #
+ # (deep breath) calculate attention and out projection
+ #
+
+ if need_weights:
+ B, Nt, E = q.shape
+ q_scaled = q / math.sqrt(E)
+
+ assert not (
+ is_causal and attn_mask is None
+ ), "FIXME: is_causal not implemented for need_weights"
+
+ if attn_mask is not None:
+ attn_output_weights = torch.baddbmm(
+ attn_mask, q_scaled, k.transpose(-2, -1)
+ )
+ else:
+ attn_output_weights = torch.bmm(q_scaled, k.transpose(-2, -1))
+ attn_output_weights = softmax(attn_output_weights, dim=-1)
+ if dropout_p > 0.0:
+ attn_output_weights = dropout(attn_output_weights, p=dropout_p)
+
+ attn_output = torch.bmm(attn_output_weights, v)
+
+ attn_output = (
+ attn_output.transpose(0, 1).contiguous().view(tgt_len * bsz, embed_dim)
+ )
+ attn_output = linear(attn_output, out_proj_weight, out_proj_bias)
+ attn_output = attn_output.view(tgt_len, bsz, attn_output.size(1))
+
+ # optionally average attention weights over heads
+ attn_output_weights = attn_output_weights.view(bsz, num_heads, tgt_len, src_len)
+ if average_attn_weights:
+ attn_output_weights = attn_output_weights.mean(dim=1)
+
+ if not is_batched:
+ # squeeze the output if input was unbatched
+ attn_output = attn_output.squeeze(1)
+ attn_output_weights = attn_output_weights.squeeze(0)
+ return attn_output, attn_output_weights
+ else:
+ # attn_mask can be either (L,S) or (N*num_heads, L, S)
+ # if attn_mask's shape is (1, L, S) we need to unsqueeze to (1, 1, L, S)
+ # in order to match the input for SDPA of (N, num_heads, L, S)
+ if attn_mask is not None:
+ if attn_mask.size(0) == 1 and attn_mask.dim() == 3:
+ attn_mask = attn_mask.unsqueeze(0)
+ else:
+ attn_mask = attn_mask.view(bsz, num_heads, -1, src_len)
+
+ q = q.view(bsz, num_heads, tgt_len, head_dim)
+ k = k.view(bsz, num_heads, src_len, head_dim)
+ v = v.view(bsz, num_heads, src_len, head_dim)
+
+ # with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=True, enable_mem_efficient=True):
+ attn_output = scaled_dot_product_attention(
+ q, k, v, attn_mask, dropout_p, is_causal
+ )
+
+ attn_output = (
+ attn_output.permute(2, 0, 1, 3).contiguous().view(bsz * tgt_len, embed_dim)
+ )
+
+ attn_output = linear(attn_output, out_proj_weight, out_proj_bias)
+ attn_output = attn_output.view(tgt_len, bsz, attn_output.size(1))
+ if not is_batched:
+ # squeeze the output if input was unbatched
+ attn_output = attn_output.squeeze(1)
+ return attn_output, None
diff --git a/GPT_SoVITS/AR/modules/patched_mha_with_cache_onnx.py b/GPT_SoVITS/AR/modules/patched_mha_with_cache_onnx.py
new file mode 100644
index 0000000000000000000000000000000000000000..14bdb550a09a2f1dac610ea653848689e6443b4d
--- /dev/null
+++ b/GPT_SoVITS/AR/modules/patched_mha_with_cache_onnx.py
@@ -0,0 +1,92 @@
+from torch.nn.functional import *
+from torch.nn.functional import (
+ _mha_shape_check,
+ _canonical_mask,
+ _none_or_dtype,
+ _in_projection_packed,
+)
+
+def multi_head_attention_forward_patched(
+ query,
+ key,
+ value,
+ embed_dim_to_check: int,
+ num_heads: int,
+ in_proj_weight,
+ in_proj_bias: Optional[Tensor],
+ bias_k: Optional[Tensor],
+ bias_v: Optional[Tensor],
+ add_zero_attn: bool,
+ dropout_p: float,
+ out_proj_weight: Tensor,
+ out_proj_bias: Optional[Tensor],
+ training: bool = True,
+ key_padding_mask: Optional[Tensor] = None,
+ need_weights: bool = True,
+ attn_mask: Optional[Tensor] = None,
+ use_separate_proj_weight: bool = False,
+ q_proj_weight: Optional[Tensor] = None,
+ k_proj_weight: Optional[Tensor] = None,
+ v_proj_weight: Optional[Tensor] = None,
+ static_k: Optional[Tensor] = None,
+ static_v: Optional[Tensor] = None,
+ average_attn_weights: bool = True,
+ is_causal: bool = False,
+ cache=None,
+) -> Tuple[Tensor, Optional[Tensor]]:
+
+ # set up shape vars
+ _, _, embed_dim = query.shape
+ attn_mask = _canonical_mask(
+ mask=attn_mask,
+ mask_name="attn_mask",
+ other_type=None,
+ other_name="",
+ target_type=query.dtype,
+ check_other=False,
+ )
+ head_dim = embed_dim // num_heads
+
+ proj_qkv = linear(query, in_proj_weight, in_proj_bias)
+ proj_qkv = proj_qkv.unflatten(-1, (3, query.size(-1))).unsqueeze(0).transpose(0, -2).squeeze(-2).contiguous()
+ q, k, v = proj_qkv[0], proj_qkv[1], proj_qkv[2]
+
+ if cache["first_infer"] == 1:
+ cache["k"][cache["stage"]] = k
+ cache["v"][cache["stage"]] = v
+ else:
+ cache["k"][cache["stage"]] = torch.cat([cache["k"][cache["stage"]][:-1], k], 0)
+ cache["v"][cache["stage"]] = torch.cat([cache["v"][cache["stage"]][:-1], v], 0)
+ k = cache["k"][cache["stage"]]
+ v = cache["v"][cache["stage"]]
+ cache["stage"] = (cache["stage"] + 1) % cache["all_stage"]
+
+ attn_mask = _canonical_mask(
+ mask=attn_mask,
+ mask_name="attn_mask",
+ other_type=None,
+ other_name="",
+ target_type=q.dtype,
+ check_other=False,
+ )
+ attn_mask = attn_mask.unsqueeze(0)
+
+ q = q.view(-1, num_heads, head_dim).transpose(0, 1)
+ k = k.view(-1, num_heads, head_dim).transpose(0, 1)
+ v = v.view(-1, num_heads, head_dim).transpose(0, 1)
+
+ dropout_p = 0.0
+ attn_mask = attn_mask.unsqueeze(0)
+ q = q.view(num_heads, -1, head_dim).unsqueeze(0)
+ k = k.view(num_heads, -1, head_dim).unsqueeze(0)
+ v = v.view(num_heads, -1, head_dim).unsqueeze(0)
+ attn_output = scaled_dot_product_attention(
+ q, k, v, attn_mask, dropout_p, is_causal
+ )
+ attn_output = (
+ attn_output.permute(2, 0, 1, 3).contiguous().view(-1, embed_dim)
+ )
+ attn_output = linear(attn_output, out_proj_weight, out_proj_bias)
+ attn_output = attn_output.view(-1, 1, attn_output.size(1))
+
+ return attn_output
diff --git a/GPT_SoVITS/AR/modules/scaling.py b/GPT_SoVITS/AR/modules/scaling.py
new file mode 100644
index 0000000000000000000000000000000000000000..9256a8cbf342b6a259c48fb8821fed0492c649fd
--- /dev/null
+++ b/GPT_SoVITS/AR/modules/scaling.py
@@ -0,0 +1,335 @@
+# Copyright 2022 Xiaomi Corp. (authors: Daniel Povey)
+#
+# See ../../../../LICENSE for clarification regarding multiple authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+import math
+import random
+from typing import Optional
+from typing import Tuple
+from typing import Union
+
+import torch
+import torch.nn as nn
+from torch import Tensor
+
+
+class DoubleSwishFunction(torch.autograd.Function):
+ """
+ double_swish(x) = x * torch.sigmoid(x-1)
+ This is a definition, originally motivated by its close numerical
+ similarity to swish(swish(x)), where swish(x) = x * sigmoid(x).
+
+ Memory-efficient derivative computation:
+ double_swish(x) = x * s, where s(x) = torch.sigmoid(x-1)
+ double_swish'(x) = d/dx double_swish(x) = x * s'(x) + x' * s(x) = x * s'(x) + s(x).
+ Now, s'(x) = s(x) * (1-s(x)).
+ double_swish'(x) = x * s'(x) + s(x).
+ = x * s(x) * (1-s(x)) + s(x).
+ = double_swish(x) * (1-s(x)) + s(x)
+ ... so we just need to remember s(x) but not x itself.
+ """
+
+ @staticmethod
+ def forward(ctx, x: Tensor) -> Tensor:
+ requires_grad = x.requires_grad
+ x_dtype = x.dtype
+ if x.dtype == torch.float16:
+ x = x.to(torch.float32)
+
+ s = torch.sigmoid(x - 1.0)
+ y = x * s
+
+ if requires_grad:
+ deriv = y * (1 - s) + s
+ # notes on derivative of x * sigmoid(x - 1):
+ # https://www.wolframalpha.com/input?i=d%2Fdx+%28x+*+sigmoid%28x-1%29%29
+ # min \simeq -0.043638. Take floor as -0.043637 so it's a lower bund
+ # max \simeq 1.1990. Take ceil to be 1.2 so it's an upper bound.
+ # the combination of "+ torch.rand_like(deriv)" and casting to torch.uint8 (which
+ # floors), should be expectation-preserving.
+ floor = -0.043637
+ ceil = 1.2
+ d_scaled = (deriv - floor) * (255.0 / (ceil - floor)) + torch.rand_like(
+ deriv
+ )
+ if __name__ == "__main__":
+ # for self-testing only.
+ assert d_scaled.min() >= 0.0
+ assert d_scaled.max() < 256.0
+ d_int = d_scaled.to(torch.uint8)
+ ctx.save_for_backward(d_int)
+ if x.dtype == torch.float16 or torch.is_autocast_enabled():
+ y = y.to(torch.float16)
+ return y
+
+ @staticmethod
+ def backward(ctx, y_grad: Tensor) -> Tensor:
+ (d,) = ctx.saved_tensors
+ # the same constants as used in forward pass.
+ floor = -0.043637
+ ceil = 1.2
+ d = d * ((ceil - floor) / 255.0) + floor
+ return y_grad * d
+
+
+class DoubleSwish(torch.nn.Module):
+ def forward(self, x: Tensor) -> Tensor:
+ """Return double-swish activation function which is an approximation to Swish(Swish(x)),
+ that we approximate closely with x * sigmoid(x-1).
+ """
+ if torch.jit.is_scripting() or torch.jit.is_tracing():
+ return x * torch.sigmoid(x - 1.0)
+ return DoubleSwishFunction.apply(x)
+
+
+class ActivationBalancerFunction(torch.autograd.Function):
+ @staticmethod
+ def forward(
+ ctx,
+ x: Tensor,
+ scale_factor: Tensor,
+ sign_factor: Optional[Tensor],
+ channel_dim: int,
+ ) -> Tensor:
+ if channel_dim < 0:
+ channel_dim += x.ndim
+ ctx.channel_dim = channel_dim
+ xgt0 = x > 0
+ if sign_factor is None:
+ ctx.save_for_backward(xgt0, scale_factor)
+ else:
+ ctx.save_for_backward(xgt0, scale_factor, sign_factor)
+ return x
+
+ @staticmethod
+ def backward(ctx, x_grad: Tensor) -> Tuple[Tensor, None, None, None]:
+ if len(ctx.saved_tensors) == 3:
+ xgt0, scale_factor, sign_factor = ctx.saved_tensors
+ for _ in range(ctx.channel_dim, x_grad.ndim - 1):
+ scale_factor = scale_factor.unsqueeze(-1)
+ sign_factor = sign_factor.unsqueeze(-1)
+ factor = sign_factor + scale_factor * (xgt0.to(x_grad.dtype) - 0.5)
+ else:
+ xgt0, scale_factor = ctx.saved_tensors
+ for _ in range(ctx.channel_dim, x_grad.ndim - 1):
+ scale_factor = scale_factor.unsqueeze(-1)
+ factor = scale_factor * (xgt0.to(x_grad.dtype) - 0.5)
+ neg_delta_grad = x_grad.abs() * factor
+ return (
+ x_grad - neg_delta_grad,
+ None,
+ None,
+ None,
+ )
+
+
+def _compute_scale_factor(
+ x: Tensor,
+ channel_dim: int,
+ min_abs: float,
+ max_abs: float,
+ gain_factor: float,
+ max_factor: float,
+) -> Tensor:
+ if channel_dim < 0:
+ channel_dim += x.ndim
+ sum_dims = [d for d in range(x.ndim) if d != channel_dim]
+ x_abs_mean = torch.mean(x.abs(), dim=sum_dims).to(torch.float32)
+
+ if min_abs == 0.0:
+ below_threshold = 0.0
+ else:
+ # below_threshold is 0 if x_abs_mean > min_abs, can be at most max_factor if
+ # x_abs)_mean , min_abs.
+ below_threshold = ((min_abs - x_abs_mean) * (gain_factor / min_abs)).clamp(
+ min=0, max=max_factor
+ )
+
+ above_threshold = ((x_abs_mean - max_abs) * (gain_factor / max_abs)).clamp(
+ min=0, max=max_factor
+ )
+
+ return below_threshold - above_threshold
+
+
+def _compute_sign_factor(
+ x: Tensor,
+ channel_dim: int,
+ min_positive: float,
+ max_positive: float,
+ gain_factor: float,
+ max_factor: float,
+) -> Tensor:
+ if channel_dim < 0:
+ channel_dim += x.ndim
+ sum_dims = [d for d in range(x.ndim) if d != channel_dim]
+ proportion_positive = torch.mean((x > 0).to(torch.float32), dim=sum_dims)
+ if min_positive == 0.0:
+ factor1 = 0.0
+ else:
+ # 0 if proportion_positive >= min_positive, else can be
+ # as large as max_factor.
+ factor1 = (
+ (min_positive - proportion_positive) * (gain_factor / min_positive)
+ ).clamp_(min=0, max=max_factor)
+
+ if max_positive == 1.0:
+ factor2 = 0.0
+ else:
+ # 0 if self.proportion_positive <= max_positive, else can be
+ # as large as -max_factor.
+ factor2 = (
+ (proportion_positive - max_positive) * (gain_factor / (1.0 - max_positive))
+ ).clamp_(min=0, max=max_factor)
+ sign_factor = factor1 - factor2
+ # require min_positive != 0 or max_positive != 1:
+ assert not isinstance(sign_factor, float)
+ return sign_factor
+
+
+class ActivationBalancer(torch.nn.Module):
+ """
+ Modifies the backpropped derivatives of a function to try to encourage, for
+ each channel, that it is positive at least a proportion `threshold` of the
+ time. It does this by multiplying negative derivative values by up to
+ (1+max_factor), and positive derivative values by up to (1-max_factor),
+ interpolated from 1 at the threshold to those extremal values when none
+ of the inputs are positive.
+
+ Args:
+ num_channels: the number of channels
+ channel_dim: the dimension/axis corresponding to the channel, e.g.
+ -1, 0, 1, 2; will be interpreted as an offset from x.ndim if negative.
+ min_positive: the minimum, per channel, of the proportion of the time
+ that (x > 0), below which we start to modify the derivatives.
+ max_positive: the maximum, per channel, of the proportion of the time
+ that (x > 0), above which we start to modify the derivatives.
+ max_factor: the maximum factor by which we modify the derivatives for
+ either the sign constraint or the magnitude constraint;
+ e.g. with max_factor=0.02, the the derivatives would be multiplied by
+ values in the range [0.98..1.02].
+ sign_gain_factor: determines the 'gain' with which we increase the
+ change in gradient once the constraints on min_positive and max_positive
+ are violated.
+ scale_gain_factor: determines the 'gain' with which we increase the
+ change in gradient once the constraints on min_abs and max_abs
+ are violated.
+ min_abs: the minimum average-absolute-value difference from the mean
+ value per channel, which we allow, before we start to modify
+ the derivatives to prevent this.
+ max_abs: the maximum average-absolute-value difference from the mean
+ value per channel, which we allow, before we start to modify
+ the derivatives to prevent this.
+ min_prob: determines the minimum probability with which we modify the
+ gradients for the {min,max}_positive and {min,max}_abs constraints,
+ on each forward(). This is done randomly to prevent all layers
+ from doing it at the same time. Early in training we may use
+ higher probabilities than this; it will decay to this value.
+ """
+
+ def __init__(
+ self,
+ num_channels: int,
+ channel_dim: int,
+ min_positive: float = 0.05,
+ max_positive: float = 0.95,
+ max_factor: float = 0.04,
+ sign_gain_factor: float = 0.01,
+ scale_gain_factor: float = 0.02,
+ min_abs: float = 0.2,
+ max_abs: float = 100.0,
+ min_prob: float = 0.1,
+ ):
+ super(ActivationBalancer, self).__init__()
+ self.num_channels = num_channels
+ self.channel_dim = channel_dim
+ self.min_positive = min_positive
+ self.max_positive = max_positive
+ self.max_factor = max_factor
+ self.min_abs = min_abs
+ self.max_abs = max_abs
+ self.min_prob = min_prob
+ self.sign_gain_factor = sign_gain_factor
+ self.scale_gain_factor = scale_gain_factor
+
+ # count measures how many times the forward() function has been called.
+ # We occasionally sync this to a tensor called `count`, that exists to
+ # make sure it is synced to disk when we load and save the model.
+ self.cpu_count = 0
+ self.register_buffer("count", torch.tensor(0, dtype=torch.int64))
+
+ def forward(self, x: Tensor) -> Tensor:
+ if torch.jit.is_scripting() or not x.requires_grad or torch.jit.is_tracing():
+ return _no_op(x)
+
+ count = self.cpu_count
+ self.cpu_count += 1
+
+ if random.random() < 0.01:
+ # Occasionally sync self.cpu_count with self.count.
+ # count affects the decay of 'prob'. don't do this on every iter,
+ # because syncing with the GPU is slow.
+ self.cpu_count = max(self.cpu_count, self.count.item())
+ self.count.fill_(self.cpu_count)
+
+ # the prob of doing some work exponentially decreases from 0.5 till it hits
+ # a floor at min_prob (==0.1, by default)
+ prob = max(self.min_prob, 0.5 ** (1 + (count / 4000.0)))
+
+ if random.random() < prob:
+ sign_gain_factor = 0.5
+ if self.min_positive != 0.0 or self.max_positive != 1.0:
+ sign_factor = _compute_sign_factor(
+ x,
+ self.channel_dim,
+ self.min_positive,
+ self.max_positive,
+ gain_factor=self.sign_gain_factor / prob,
+ max_factor=self.max_factor,
+ )
+ else:
+ sign_factor = None
+
+ scale_factor = _compute_scale_factor(
+ x.detach(),
+ self.channel_dim,
+ min_abs=self.min_abs,
+ max_abs=self.max_abs,
+ gain_factor=self.scale_gain_factor / prob,
+ max_factor=self.max_factor,
+ )
+ return ActivationBalancerFunction.apply(
+ x,
+ scale_factor,
+ sign_factor,
+ self.channel_dim,
+ )
+ else:
+ return _no_op(x)
+
+
+def BalancedDoubleSwish(
+ d_model, channel_dim=-1, max_abs=10.0, min_prob=0.25
+) -> nn.Sequential:
+ """
+ ActivationBalancer -> DoubleSwish
+ """
+ balancer = ActivationBalancer(
+ d_model, channel_dim=channel_dim, max_abs=max_abs, min_prob=min_prob
+ )
+ return nn.Sequential(
+ balancer,
+ DoubleSwish(),
+ )
diff --git a/GPT_SoVITS/AR/modules/transformer.py b/GPT_SoVITS/AR/modules/transformer.py
new file mode 100644
index 0000000000000000000000000000000000000000..7921f48e70bd6849897389d8e0a39d1ac4062b97
--- /dev/null
+++ b/GPT_SoVITS/AR/modules/transformer.py
@@ -0,0 +1,378 @@
+# modified from https://github.com/lifeiteng/vall-e/blob/main/valle/modules/transformer.py
+import copy
+import numbers
+from functools import partial
+from typing import Any
+from typing import Callable
+from typing import List
+from typing import Optional
+from typing import Tuple
+from typing import Union
+
+import torch
+from AR.modules.activation import MultiheadAttention
+from AR.modules.scaling import BalancedDoubleSwish
+from torch import nn
+from torch import Tensor
+from torch.nn import functional as F
+
+_shape_t = Union[int, List[int], torch.Size]
+
+
+class LayerNorm(nn.Module):
+ __constants__ = ["normalized_shape", "eps", "elementwise_affine"]
+ normalized_shape: Tuple[int, ...]
+ eps: float
+ elementwise_affine: bool
+
+ def __init__(
+ self,
+ normalized_shape: _shape_t,
+ eps: float = 1e-5,
+ elementwise_affine: bool = True,
+ device=None,
+ dtype=None,
+ ) -> None:
+ factory_kwargs = {"device": device, "dtype": dtype}
+ super(LayerNorm, self).__init__()
+ if isinstance(normalized_shape, numbers.Integral):
+ # mypy error: incompatible types in assignment
+ normalized_shape = (normalized_shape,) # type: ignore[assignment]
+ self.normalized_shape = tuple(normalized_shape) # type: ignore[arg-type]
+ self.eps = eps
+ self.elementwise_affine = elementwise_affine
+ if self.elementwise_affine:
+ self.weight = nn.Parameter(
+ torch.empty(self.normalized_shape, **factory_kwargs)
+ )
+ self.bias = nn.Parameter(
+ torch.empty(self.normalized_shape, **factory_kwargs)
+ )
+ else:
+ self.register_parameter("weight", None)
+ self.register_parameter("bias", None)
+
+ self.reset_parameters()
+
+ def reset_parameters(self) -> None:
+ if self.elementwise_affine:
+ nn.init.ones_(self.weight)
+ nn.init.zeros_(self.bias)
+
+ def forward(self, input: Tensor, embedding: Any = None) -> Tensor:
+ if isinstance(input, tuple):
+ input, embedding = input
+ return (
+ F.layer_norm(
+ input,
+ self.normalized_shape,
+ self.weight,
+ self.bias,
+ self.eps,
+ ),
+ embedding,
+ )
+
+ assert embedding is None
+ return F.layer_norm(
+ input, self.normalized_shape, self.weight, self.bias, self.eps
+ )
+
+ def extra_repr(self) -> str:
+ return (
+ "{normalized_shape}, eps={eps}, "
+ "elementwise_affine={elementwise_affine}".format(**self.__dict__)
+ )
+
+
+class IdentityNorm(nn.Module):
+ def __init__(
+ self,
+ d_model: int,
+ eps: float = 1e-5,
+ device=None,
+ dtype=None,
+ ) -> None:
+ super(IdentityNorm, self).__init__()
+
+ def forward(self, input: Tensor, embedding: Any = None) -> Tensor:
+ if isinstance(input, tuple):
+ return input
+
+ assert embedding is None
+ return input
+
+
+class TransformerEncoder(nn.Module):
+ r"""TransformerEncoder is a stack of N encoder layers. Users can build the
+ BERT(https://arxiv.org/abs/1810.04805) model with corresponding parameters.
+
+ Args:
+ encoder_layer: an instance of the TransformerEncoderLayer() class (required).
+ num_layers: the number of sub-encoder-layers in the encoder (required).
+ norm: the layer normalization component (optional).
+ enable_nested_tensor: if True, input will automatically convert to nested tensor
+ (and convert back on output). This will improve the overall performance of
+ TransformerEncoder when padding rate is high. Default: ``True`` (enabled).
+
+ Examples::
+ >>> encoder_layer = TransformerEncoderLayer(d_model=512, nhead=8)
+ >>> transformer_encoder = TransformerEncoder(encoder_layer, num_layers=6)
+ >>> src = torch.rand(10, 32, 512)
+ >>> out = transformer_encoder(src)
+ """
+ __constants__ = ["norm"]
+
+ def __init__(self, encoder_layer, num_layers, norm=None):
+ super(TransformerEncoder, self).__init__()
+ self.layers = _get_clones(encoder_layer, num_layers)
+ self.num_layers = num_layers
+ self.norm = norm
+
+ def forward(
+ self,
+ src: Tensor,
+ mask: Optional[Tensor] = None,
+ src_key_padding_mask: Optional[Tensor] = None,
+ return_layer_states: bool = False,
+ cache=None,
+ ) -> Tensor:
+ r"""Pass the input through the encoder layers in turn.
+
+ Args:
+ src: the sequence to the encoder (required).
+ mask: the mask for the src sequence (optional).
+ src_key_padding_mask: the mask for the src keys per batch (optional).
+ return_layer_states: return layers' state (optional).
+
+ Shape:
+ see the docs in Transformer class.
+ """
+ if return_layer_states:
+ layer_states = [] # layers' output
+ output = src
+ for mod in self.layers:
+ output = mod(
+ output,
+ src_mask=mask,
+ src_key_padding_mask=src_key_padding_mask,
+ cache=cache,
+ )
+ layer_states.append(output[0])
+
+ if self.norm is not None:
+ output = self.norm(output)
+
+ return layer_states, output
+
+ output = src
+ for mod in self.layers:
+ output = mod(
+ output,
+ src_mask=mask,
+ src_key_padding_mask=src_key_padding_mask,
+ cache=cache,
+ )
+
+ if self.norm is not None:
+ output = self.norm(output)
+
+ return output
+
+
+class TransformerEncoderLayer(nn.Module):
+ __constants__ = ["batch_first", "norm_first"]
+
+ def __init__(
+ self,
+ d_model: int,
+ nhead: int,
+ dim_feedforward: int = 2048,
+ dropout: float = 0.1,
+ activation: Union[str, Callable[[Tensor], Tensor]] = F.relu,
+ batch_first: bool = False,
+ norm_first: bool = False,
+ device=None,
+ dtype=None,
+ linear1_self_attention_cls: nn.Module = nn.Linear,
+ linear2_self_attention_cls: nn.Module = nn.Linear,
+ linear1_feedforward_cls: nn.Module = nn.Linear,
+ linear2_feedforward_cls: nn.Module = nn.Linear,
+ layer_norm_cls: nn.Module = LayerNorm,
+ layer_norm_eps: float = 1e-5,
+ adaptive_layer_norm=False,
+ ) -> None:
+ factory_kwargs = {"device": device, "dtype": dtype}
+ super(TransformerEncoderLayer, self).__init__()
+ # print(233333333333,d_model,nhead)
+ # import os
+ # os._exit(2333333)
+ self.self_attn = MultiheadAttention(
+ d_model, # 512 16
+ nhead,
+ dropout=dropout,
+ batch_first=batch_first,
+ linear1_cls=linear1_self_attention_cls,
+ linear2_cls=linear2_self_attention_cls,
+ **factory_kwargs,
+ )
+
+ # Implementation of Feedforward model
+ self.linear1 = linear1_feedforward_cls(
+ d_model, dim_feedforward, **factory_kwargs
+ )
+ self.dropout = nn.Dropout(dropout)
+ self.linear2 = linear2_feedforward_cls(
+ dim_feedforward, d_model, **factory_kwargs
+ )
+
+ self.norm_first = norm_first
+ self.dropout1 = nn.Dropout(dropout)
+ self.dropout2 = nn.Dropout(dropout)
+
+ # Legacy string support for activation function.
+ if isinstance(activation, str):
+ activation = _get_activation_fn(activation)
+ elif isinstance(activation, partial):
+ activation = activation(d_model)
+ elif activation == BalancedDoubleSwish:
+ activation = BalancedDoubleSwish(d_model)
+
+ # # We can't test self.activation in forward() in TorchScript,
+ # # so stash some information about it instead.
+ # if activation is F.relu or isinstance(activation, torch.nn.ReLU):
+ # self.activation_relu_or_gelu = 1
+ # elif activation is F.gelu or isinstance(activation, torch.nn.GELU):
+ # self.activation_relu_or_gelu = 2
+ # else:
+ # self.activation_relu_or_gelu = 0
+ self.activation = activation
+
+ norm1 = layer_norm_cls(d_model, eps=layer_norm_eps, **factory_kwargs)
+ if layer_norm_cls == IdentityNorm:
+ norm2 = BalancedBasicNorm(d_model, eps=layer_norm_eps, **factory_kwargs)
+ else:
+ norm2 = layer_norm_cls(d_model, eps=layer_norm_eps, **factory_kwargs)
+
+ if adaptive_layer_norm:
+ self.norm1 = AdaptiveLayerNorm(d_model, norm1)
+ self.norm2 = AdaptiveLayerNorm(d_model, norm2)
+ else:
+ self.norm1 = norm1
+ self.norm2 = norm2
+
+ def __setstate__(self, state):
+ super(TransformerEncoderLayer, self).__setstate__(state)
+ if not hasattr(self, "activation"):
+ self.activation = F.relu
+
+ def forward(
+ self,
+ src: Tensor,
+ src_mask: Optional[Tensor] = None,
+ src_key_padding_mask: Optional[Tensor] = None,
+ cache=None,
+ ) -> Tensor:
+ r"""Pass the input through the encoder layer.
+
+ Args:
+ src: the sequence to the encoder layer (required).
+ src_mask: the mask for the src sequence (optional).
+ src_key_padding_mask: the mask for the src keys per batch (optional).
+
+ Shape:
+ see the docs in Transformer class.
+ """
+ x, stage_embedding = src, None
+ is_src_tuple = False
+ if isinstance(src, tuple):
+ x, stage_embedding = src
+ is_src_tuple = True
+
+ if src_key_padding_mask is not None:
+ _skpm_dtype = src_key_padding_mask.dtype
+ if _skpm_dtype != torch.bool and not torch.is_floating_point(
+ src_key_padding_mask
+ ):
+ raise AssertionError(
+ "only bool and floating types of key_padding_mask are supported"
+ )
+
+ if self.norm_first:
+ x = x + self._sa_block(
+ self.norm1(x, stage_embedding),
+ src_mask,
+ src_key_padding_mask,
+ cache=cache,
+ )
+ x = x + self._ff_block(self.norm2(x, stage_embedding))
+ else:
+ x = self.norm1(
+ x + self._sa_block(x, src_mask, src_key_padding_mask, cache=cache),
+ stage_embedding,
+ )
+ x = self.norm2(x + self._ff_block(x), stage_embedding)
+
+ if is_src_tuple:
+ return (x, stage_embedding)
+ return x
+
+ # self-attention block
+ def _sa_block(
+ self,
+ x: Tensor,
+ attn_mask: Optional[Tensor],
+ key_padding_mask: Optional[Tensor],
+ cache=None,
+ ) -> Tensor:
+ # print(x.shape,attn_mask.shape,key_padding_mask)
+ # torch.Size([1, 188, 512]) torch.Size([188, 188]) None
+ # import os
+ # os._exit(23333)
+ x = self.self_attn(
+ x,
+ x,
+ x,
+ attn_mask=attn_mask,
+ key_padding_mask=key_padding_mask,
+ need_weights=False,
+ cache=cache,
+ )[0]
+ return self.dropout1(x)
+
+ # feed forward block
+ def _ff_block(self, x: Tensor) -> Tensor:
+ x = self.linear2(self.dropout(self.activation(self.linear1(x))))
+ return self.dropout2(x)
+
+
+class AdaptiveLayerNorm(nn.Module):
+ r"""Adaptive Layer Normalization"""
+
+ def __init__(self, d_model, norm) -> None:
+ super(AdaptiveLayerNorm, self).__init__()
+ self.project_layer = nn.Linear(d_model, 2 * d_model)
+ self.norm = norm
+ self.d_model = d_model
+ self.eps = self.norm.eps
+
+ def forward(self, input: Tensor, embedding: Tensor = None) -> Tensor:
+ if isinstance(input, tuple):
+ input, embedding = input
+ weight, bias = torch.split(
+ self.project_layer(embedding),
+ split_size_or_sections=self.d_model,
+ dim=-1,
+ )
+ return (weight * self.norm(input) + bias, embedding)
+
+ weight, bias = torch.split(
+ self.project_layer(embedding),
+ split_size_or_sections=self.d_model,
+ dim=-1,
+ )
+ return weight * self.norm(input) + bias
+
+
+def _get_clones(module, N):
+ return nn.ModuleList([copy.deepcopy(module) for i in range(N)])
diff --git a/GPT_SoVITS/AR/modules/transformer_onnx.py b/GPT_SoVITS/AR/modules/transformer_onnx.py
new file mode 100644
index 0000000000000000000000000000000000000000..a3f68b43e7a4f3c8d989140009477e07a2d44d19
--- /dev/null
+++ b/GPT_SoVITS/AR/modules/transformer_onnx.py
@@ -0,0 +1,292 @@
+# modified from https://github.com/lifeiteng/vall-e/blob/main/valle/modules/transformer.py
+import copy
+import numbers
+from functools import partial
+from typing import Any
+from typing import Callable
+from typing import List
+from typing import Optional
+from typing import Tuple
+from typing import Union
+
+import torch
+from AR.modules.activation_onnx import MultiheadAttention
+from AR.modules.scaling import BalancedDoubleSwish
+from torch import nn
+from torch import Tensor
+from torch.nn import functional as F
+
+_shape_t = Union[int, List[int], torch.Size]
+
+
+class LayerNorm(nn.Module):
+ __constants__ = ["normalized_shape", "eps", "elementwise_affine"]
+ normalized_shape: Tuple[int, ...]
+ eps: float
+ elementwise_affine: bool
+
+ def __init__(
+ self,
+ normalized_shape: _shape_t,
+ eps: float = 1e-5,
+ elementwise_affine: bool = True,
+ device=None,
+ dtype=None,
+ ) -> None:
+ factory_kwargs = {"device": device, "dtype": dtype}
+ super(LayerNorm, self).__init__()
+ if isinstance(normalized_shape, numbers.Integral):
+ # mypy error: incompatible types in assignment
+ normalized_shape = (normalized_shape,) # type: ignore[assignment]
+ self.normalized_shape = tuple(normalized_shape) # type: ignore[arg-type]
+ self.eps = eps
+ self.elementwise_affine = elementwise_affine
+ if self.elementwise_affine:
+ self.weight = nn.Parameter(
+ torch.empty(self.normalized_shape, **factory_kwargs)
+ )
+ self.bias = nn.Parameter(
+ torch.empty(self.normalized_shape, **factory_kwargs)
+ )
+ else:
+ self.register_parameter("weight", None)
+ self.register_parameter("bias", None)
+
+ self.reset_parameters()
+
+ def reset_parameters(self) -> None:
+ if self.elementwise_affine:
+ nn.init.ones_(self.weight)
+ nn.init.zeros_(self.bias)
+
+ def forward(self, input: Tensor, embedding: Any = None) -> Tensor:
+ if isinstance(input, tuple):
+ input, embedding = input
+ return (
+ F.layer_norm(
+ input,
+ self.normalized_shape,
+ self.weight,
+ self.bias,
+ self.eps,
+ ),
+ embedding,
+ )
+
+ assert embedding is None
+ return F.layer_norm(
+ input, self.normalized_shape, self.weight, self.bias, self.eps
+ )
+
+ def extra_repr(self) -> str:
+ return (
+ "{normalized_shape}, eps={eps}, "
+ "elementwise_affine={elementwise_affine}".format(**self.__dict__)
+ )
+
+
+class IdentityNorm(nn.Module):
+ def __init__(
+ self,
+ d_model: int,
+ eps: float = 1e-5,
+ device=None,
+ dtype=None,
+ ) -> None:
+ super(IdentityNorm, self).__init__()
+
+ def forward(self, input: Tensor, embedding: Any = None) -> Tensor:
+ if isinstance(input, tuple):
+ return input
+
+ assert embedding is None
+ return input
+
+
+class TransformerEncoder(nn.Module):
+ r"""TransformerEncoder is a stack of N encoder layers. Users can build the
+ BERT(https://arxiv.org/abs/1810.04805) model with corresponding parameters.
+
+ Args:
+ encoder_layer: an instance of the TransformerEncoderLayer() class (required).
+ num_layers: the number of sub-encoder-layers in the encoder (required).
+ norm: the layer normalization component (optional).
+ enable_nested_tensor: if True, input will automatically convert to nested tensor
+ (and convert back on output). This will improve the overall performance of
+ TransformerEncoder when padding rate is high. Default: ``True`` (enabled).
+
+ Examples::
+ >>> encoder_layer = TransformerEncoderLayer(d_model=512, nhead=8)
+ >>> transformer_encoder = TransformerEncoder(encoder_layer, num_layers=6)
+ >>> src = torch.rand(10, 32, 512)
+ >>> out = transformer_encoder(src)
+ """
+ __constants__ = ["norm"]
+
+ def __init__(self, encoder_layer, num_layers, norm=None):
+ super(TransformerEncoder, self).__init__()
+ self.layers = _get_clones(encoder_layer, num_layers)
+ self.num_layers = num_layers
+ self.norm = norm
+
+ def forward(
+ self,
+ src: Tensor,
+ mask: Optional[Tensor] = None,
+ src_key_padding_mask: Optional[Tensor] = None,
+ return_layer_states: bool = False,
+ cache=None,
+ ) -> Tensor:
+ output = src
+ for mod in self.layers:
+ output = mod(
+ output,
+ src_mask=mask,
+ src_key_padding_mask=src_key_padding_mask,
+ cache=cache,
+ )
+
+ if self.norm is not None:
+ output = self.norm(output)
+
+ return output
+
+
+class TransformerEncoderLayer(nn.Module):
+ __constants__ = ["batch_first", "norm_first"]
+ def __init__(
+ self,
+ d_model: int,
+ nhead: int,
+ dim_feedforward: int = 2048,
+ dropout: float = 0.1,
+ activation: Union[str, Callable[[Tensor], Tensor]] = F.relu,
+ batch_first: bool = False,
+ norm_first: bool = False,
+ device=None,
+ dtype=None,
+ linear1_self_attention_cls: nn.Module = nn.Linear,
+ linear2_self_attention_cls: nn.Module = nn.Linear,
+ linear1_feedforward_cls: nn.Module = nn.Linear,
+ linear2_feedforward_cls: nn.Module = nn.Linear,
+ layer_norm_cls: nn.Module = LayerNorm,
+ layer_norm_eps: float = 1e-5,
+ adaptive_layer_norm=False,
+ ) -> None:
+ factory_kwargs = {"device": device, "dtype": dtype}
+ super(TransformerEncoderLayer, self).__init__()
+ self.self_attn = MultiheadAttention(
+ d_model, # 512 16
+ nhead,
+ dropout=dropout,
+ batch_first=batch_first,
+ linear1_cls=linear1_self_attention_cls,
+ linear2_cls=linear2_self_attention_cls,
+ **factory_kwargs,
+ )
+ self.linear1 = linear1_feedforward_cls(
+ d_model, dim_feedforward, **factory_kwargs
+ )
+ self.dropout = nn.Dropout(dropout)
+ self.linear2 = linear2_feedforward_cls(
+ dim_feedforward, d_model, **factory_kwargs
+ )
+ self.norm_first = norm_first
+ self.dropout1 = nn.Dropout(dropout)
+ self.dropout2 = nn.Dropout(dropout)
+ if isinstance(activation, str):
+ activation = _get_activation_fn(activation)
+ elif isinstance(activation, partial):
+ activation = activation(d_model)
+ elif activation == BalancedDoubleSwish:
+ activation = BalancedDoubleSwish(d_model)
+ self.activation = activation
+
+ norm1 = layer_norm_cls(d_model, eps=layer_norm_eps, **factory_kwargs)
+ if layer_norm_cls == IdentityNorm:
+ norm2 = BalancedBasicNorm(d_model, eps=layer_norm_eps, **factory_kwargs)
+ else:
+ norm2 = layer_norm_cls(d_model, eps=layer_norm_eps, **factory_kwargs)
+
+ if adaptive_layer_norm:
+ self.norm1 = AdaptiveLayerNorm(d_model, norm1)
+ self.norm2 = AdaptiveLayerNorm(d_model, norm2)
+ else:
+ self.norm1 = norm1
+ self.norm2 = norm2
+
+ def __setstate__(self, state):
+ super(TransformerEncoderLayer, self).__setstate__(state)
+ if not hasattr(self, "activation"):
+ self.activation = F.relu
+
+ def forward(
+ self,
+ src: Tensor,
+ src_mask: Optional[Tensor] = None,
+ src_key_padding_mask: Optional[Tensor] = None,
+ cache=None,
+ ) -> Tensor:
+ x = src
+ stage_embedding = None
+ x = self.norm1(
+ x + self._sa_block(x, src_mask, src_key_padding_mask, cache=cache),
+ stage_embedding,
+ )
+ x = self.norm2(x + self._ff_block(x), stage_embedding)
+
+ return x
+
+ def _sa_block(
+ self,
+ x: Tensor,
+ attn_mask: Optional[Tensor],
+ key_padding_mask: Optional[Tensor],
+ cache=None,
+ ) -> Tensor:
+ x = self.self_attn(
+ x,
+ x,
+ x,
+ attn_mask=attn_mask,
+ key_padding_mask=key_padding_mask,
+ need_weights=False,
+ cache=cache,
+ )
+ return self.dropout1(x)
+
+ def _ff_block(self, x: Tensor) -> Tensor:
+ x = self.linear2(self.dropout(self.activation(self.linear1(x))))
+ return self.dropout2(x)
+
+
+class AdaptiveLayerNorm(nn.Module):
+ r"""Adaptive Layer Normalization"""
+
+ def __init__(self, d_model, norm) -> None:
+ super(AdaptiveLayerNorm, self).__init__()
+ self.project_layer = nn.Linear(d_model, 2 * d_model)
+ self.norm = norm
+ self.d_model = d_model
+ self.eps = self.norm.eps
+
+ def forward(self, input: Tensor, embedding: Tensor = None) -> Tensor:
+ if isinstance(input, tuple):
+ input, embedding = input
+ weight, bias = torch.split(
+ self.project_layer(embedding),
+ split_size_or_sections=self.d_model,
+ dim=-1,
+ )
+ return (weight * self.norm(input) + bias, embedding)
+
+ weight, bias = torch.split(
+ self.project_layer(embedding),
+ split_size_or_sections=self.d_model,
+ dim=-1,
+ )
+ return weight * self.norm(input) + bias
+
+
+def _get_clones(module, N):
+ return nn.ModuleList([copy.deepcopy(module) for i in range(N)])
diff --git a/GPT_SoVITS/AR/text_processing/__init__.py b/GPT_SoVITS/AR/text_processing/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/GPT_SoVITS/AR/text_processing/phonemizer.py b/GPT_SoVITS/AR/text_processing/phonemizer.py
new file mode 100644
index 0000000000000000000000000000000000000000..9c5f58fb74da836764cc9d71b8556e979f2b2830
--- /dev/null
+++ b/GPT_SoVITS/AR/text_processing/phonemizer.py
@@ -0,0 +1,79 @@
+# modified from https://github.com/yangdongchao/SoundStorm/blob/master/soundstorm/s1/AR/text_processing/phonemizer.py
+# reference: https://github.com/lifeiteng/vall-e
+import itertools
+import re
+from typing import Dict
+from typing import List
+
+import regex
+from gruut import sentences
+from gruut.const import Sentence
+from gruut.const import Word
+from AR.text_processing.symbols import SYMBOL_TO_ID
+
+
+class GruutPhonemizer:
+ def __init__(self, language: str):
+ self._phonemizer = sentences
+ self.lang = language
+ self.symbol_to_id = SYMBOL_TO_ID
+ self._special_cases_dict: Dict[str] = {
+ r"\.\.\.": "... ",
+ ";": "; ",
+ ":": ": ",
+ ",": ", ",
+ r"\.": ". ",
+ "!": "! ",
+ r"\?": "? ",
+ "—": "—",
+ "…": "… ",
+ "«": "«",
+ "»": "»",
+ }
+ self._punctuation_regexp: str = (
+ rf"([{''.join(self._special_cases_dict.keys())}])"
+ )
+
+ def _normalize_punctuation(self, text: str) -> str:
+ text = regex.sub(rf"\pZ+{self._punctuation_regexp}", r"\1", text)
+ text = regex.sub(rf"{self._punctuation_regexp}(\pL)", r"\1 \2", text)
+ text = regex.sub(r"\pZ+", r" ", text)
+ return text.strip()
+
+ def _convert_punctuation(self, word: Word) -> str:
+ if not word.phonemes:
+ return ""
+ if word.phonemes[0] in ["‖", "|"]:
+ return word.text.strip()
+
+ phonemes = "".join(word.phonemes)
+ # remove modifier characters ˈˌː with regex
+ phonemes = re.sub(r"[ˈˌː͡]", "", phonemes)
+ return phonemes.strip()
+
+ def phonemize(self, text: str, espeak: bool = False) -> str:
+ text_to_phonemize: str = self._normalize_punctuation(text)
+ sents: List[Sentence] = [
+ sent
+ for sent in self._phonemizer(text_to_phonemize, lang="en-us", espeak=espeak)
+ ]
+ words: List[str] = [
+ self._convert_punctuation(word) for word in itertools.chain(*sents)
+ ]
+ return " ".join(words)
+
+ def transform(self, phonemes):
+ # convert phonemes to ids
+ # dictionary is in symbols.py
+ return [self.symbol_to_id[p] for p in phonemes if p in self.symbol_to_id.keys()]
+
+
+if __name__ == "__main__":
+ phonemizer = GruutPhonemizer("en-us")
+ # text -> IPA
+ phonemes = phonemizer.phonemize("Hello, wor-ld ?")
+ print("phonemes:", phonemes)
+ print("len(phonemes):", len(phonemes))
+ phoneme_ids = phonemizer.transform(phonemes)
+ print("phoneme_ids:", phoneme_ids)
+ print("len(phoneme_ids):", len(phoneme_ids))
diff --git a/GPT_SoVITS/AR/text_processing/symbols.py b/GPT_SoVITS/AR/text_processing/symbols.py
new file mode 100644
index 0000000000000000000000000000000000000000..7d754a78b1fd5b3d89768585e1891404bb318118
--- /dev/null
+++ b/GPT_SoVITS/AR/text_processing/symbols.py
@@ -0,0 +1,10 @@
+# modified from https://github.com/yangdongchao/SoundStorm/blob/master/soundstorm/s1/AR/text_processing/symbols.py
+# reference: https://github.com/lifeiteng/vall-e
+PAD = "_"
+PUNCTUATION = ';:,.!?¡¿—…"«»“” '
+LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+IPA_LETTERS = "ɑɐɒæɓʙβɔɕçɗɖðʤəɘɚɛɜɝɞɟʄɡɠɢʛɦɧħɥʜɨɪʝɭɬɫɮʟɱɯɰŋɳɲɴøɵɸθœɶʘɹɺɾɻʀʁɽʂʃʈʧʉʊʋⱱʌɣɤʍχʎʏʑʐʒʔʡʕʢǀǁǂǃˈˌːˑʼʴʰʱʲʷˠˤ˞↓↑→↗↘'̩'ᵻ"
+SYMBOLS = [PAD] + list(PUNCTUATION) + list(LETTERS) + list(IPA_LETTERS)
+SPACE_ID = SYMBOLS.index(" ")
+SYMBOL_TO_ID = {s: i for i, s in enumerate(SYMBOLS)}
+ID_TO_SYMBOL = {i: s for i, s in enumerate(SYMBOLS)}
diff --git a/GPT_SoVITS/AR/utils/__init__.py b/GPT_SoVITS/AR/utils/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..c2eaf61adcfee96d6e7ec8fd70a7603e18afb567
--- /dev/null
+++ b/GPT_SoVITS/AR/utils/__init__.py
@@ -0,0 +1,37 @@
+import re
+
+
+def str2bool(str):
+ return True if str.lower() == 'true' else False
+
+
+def get_newest_ckpt(string_list):
+ # 定义一个正则表达式模式,用于匹配字符串中的数字
+ pattern = r'epoch=(\d+)-step=(\d+)\.ckpt'
+
+ # 使用正则表达式提取每个字符串中的数字信息,并创建一个包含元组的列表
+ extracted_info = []
+ for string in string_list:
+ match = re.match(pattern, string)
+ if match:
+ epoch = int(match.group(1))
+ step = int(match.group(2))
+ extracted_info.append((epoch, step, string))
+ # 按照 epoch 后面的数字和 step 后面的数字进行排序
+ sorted_info = sorted(
+ extracted_info, key=lambda x: (x[0], x[1]), reverse=True)
+ # 获取最新的 ckpt 文件名
+ newest_ckpt = sorted_info[0][2]
+ return newest_ckpt
+
+
+# 文本存在且不为空时 return True
+def check_txt_file(file_path):
+ try:
+ with open(file_path, 'r') as file:
+ text = file.readline().strip()
+ assert text.strip() != ''
+ return text
+ except Exception:
+ return False
+ return False
diff --git a/GPT_SoVITS/AR/utils/initialize.py b/GPT_SoVITS/AR/utils/initialize.py
new file mode 100644
index 0000000000000000000000000000000000000000..17ff9f92e51c8941973139d6e34d4a1c7cd8daaa
--- /dev/null
+++ b/GPT_SoVITS/AR/utils/initialize.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+"""Initialize modules for espnet2 neural networks."""
+import torch
+from typeguard import check_argument_types
+
+
+def initialize(model: torch.nn.Module, init: str):
+ """Initialize weights of a neural network module.
+
+ Parameters are initialized using the given method or distribution.
+
+ Custom initialization routines can be implemented into submodules
+ as function `espnet_initialization_fn` within the custom module.
+
+ Args:
+ model: Target.
+ init: Method of initialization.
+ """
+ assert check_argument_types()
+ print("init with", init)
+
+ # weight init
+ for p in model.parameters():
+ if p.dim() > 1:
+ if init == "xavier_uniform":
+ torch.nn.init.xavier_uniform_(p.data)
+ elif init == "xavier_normal":
+ torch.nn.init.xavier_normal_(p.data)
+ elif init == "kaiming_uniform":
+ torch.nn.init.kaiming_uniform_(p.data, nonlinearity="relu")
+ elif init == "kaiming_normal":
+ torch.nn.init.kaiming_normal_(p.data, nonlinearity="relu")
+ else:
+ raise ValueError("Unknown initialization: " + init)
+ # bias init
+ for name, p in model.named_parameters():
+ if ".bias" in name and p.dim() == 1:
+ p.data.zero_()
diff --git a/GPT_SoVITS/AR/utils/io.py b/GPT_SoVITS/AR/utils/io.py
new file mode 100644
index 0000000000000000000000000000000000000000..52f1f3c991506a9ea5d3fdbc71ded61e914694f1
--- /dev/null
+++ b/GPT_SoVITS/AR/utils/io.py
@@ -0,0 +1,34 @@
+import sys
+
+import torch
+import yaml
+
+
+def load_yaml_config(path):
+ with open(path) as f:
+ config = yaml.full_load(f)
+ return config
+
+
+def save_config_to_yaml(config, path):
+ assert path.endswith(".yaml")
+ with open(path, "w") as f:
+ f.write(yaml.dump(config))
+ f.close()
+
+
+def write_args(args, path):
+ args_dict = dict(
+ (name, getattr(args, name)) for name in dir(args) if not name.startswith("_")
+ )
+ with open(path, "a") as args_file:
+ args_file.write("==> torch version: {}\n".format(torch.__version__))
+ args_file.write(
+ "==> cudnn version: {}\n".format(torch.backends.cudnn.version())
+ )
+ args_file.write("==> Cmd:\n")
+ args_file.write(str(sys.argv))
+ args_file.write("\n==> args:\n")
+ for k, v in sorted(args_dict.items()):
+ args_file.write(" %s: %s\n" % (str(k), str(v)))
+ args_file.close()
diff --git a/GPT_SoVITS/configs/s1.yaml b/GPT_SoVITS/configs/s1.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..f8ae17d4415cbbacca2d30e1403239ecc554e98b
--- /dev/null
+++ b/GPT_SoVITS/configs/s1.yaml
@@ -0,0 +1,31 @@
+train:
+ seed: 1234
+ epochs: 300
+ batch_size: 8
+ gradient_accumulation: 4
+ save_every_n_epoch: 1
+ precision: 16
+ gradient_clip: 1.0
+optimizer:
+ lr: 0.01
+ lr_init: 0.00001
+ lr_end: 0.0001
+ warmup_steps: 2000
+ decay_steps: 40000
+data:
+ max_eval_sample: 8
+ max_sec: 54
+ num_workers: 1
+ pad_val: 1024 # same with EOS in model
+model:
+ vocab_size: 1025
+ phoneme_vocab_size: 512
+ embedding_dim: 512
+ hidden_dim: 512
+ head: 16
+ linear_units: 2048
+ n_layer: 12
+ dropout: 0
+ EOS: 1024
+inference:
+ top_k: 5
diff --git a/GPT_SoVITS/configs/s1big.yaml b/GPT_SoVITS/configs/s1big.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a811150de4c418be57b3bb932a7fe12cc7b6f679
--- /dev/null
+++ b/GPT_SoVITS/configs/s1big.yaml
@@ -0,0 +1,31 @@
+train:
+ seed: 1234
+ epochs: 300
+ batch_size: 8
+ gradient_accumulation: 4
+ save_every_n_epoch: 1
+ precision: 16-mixed
+ gradient_clip: 1.0
+optimizer:
+ lr: 0.01
+ lr_init: 0.00001
+ lr_end: 0.0001
+ warmup_steps: 2000
+ decay_steps: 40000
+data:
+ max_eval_sample: 8
+ max_sec: 54
+ num_workers: 1
+ pad_val: 1024 # same with EOS in model
+model:
+ vocab_size: 1025
+ phoneme_vocab_size: 512
+ embedding_dim: 1024
+ hidden_dim: 1024
+ head: 16
+ linear_units: 2048
+ n_layer: 16
+ dropout: 0
+ EOS: 1024
+inference:
+ top_k: 5
diff --git a/GPT_SoVITS/configs/s1big2.yaml b/GPT_SoVITS/configs/s1big2.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b8b889babbc43ada8c17cee52db68fd484cc0c97
--- /dev/null
+++ b/GPT_SoVITS/configs/s1big2.yaml
@@ -0,0 +1,31 @@
+train:
+ seed: 1234
+ epochs: 300
+ batch_size: 12
+ gradient_accumulation: 4
+ save_every_n_epoch: 1
+ precision: 16-mixed
+ gradient_clip: 1.0
+optimizer:
+ lr: 0.01
+ lr_init: 0.00001
+ lr_end: 0.0001
+ warmup_steps: 2000
+ decay_steps: 40000
+data:
+ max_eval_sample: 8
+ max_sec: 54
+ num_workers: 1
+ pad_val: 1024 # same with EOS in model
+model:
+ vocab_size: 1025
+ phoneme_vocab_size: 512
+ embedding_dim: 1024
+ hidden_dim: 1024
+ head: 16
+ linear_units: 2048
+ n_layer: 6
+ dropout: 0
+ EOS: 1024
+inference:
+ top_k: 5
diff --git a/GPT_SoVITS/configs/s1longer.yaml b/GPT_SoVITS/configs/s1longer.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..3f57abd21a85a36107d247488e988797f0f8d484
--- /dev/null
+++ b/GPT_SoVITS/configs/s1longer.yaml
@@ -0,0 +1,31 @@
+train:
+ seed: 1234
+ epochs: 20
+ batch_size: 8
+ save_every_n_epoch: 1
+ precision: 16-mixed
+ gradient_clip: 1.0
+optimizer:
+ lr: 0.01
+ lr_init: 0.00001
+ lr_end: 0.0001
+ warmup_steps: 2000
+ decay_steps: 40000
+data:
+ max_eval_sample: 8
+ max_sec: 54
+ num_workers: 4
+ pad_val: 1024 # same with EOS in model
+model:
+ vocab_size: 1025
+ phoneme_vocab_size: 512
+ embedding_dim: 512
+ hidden_dim: 512
+ head: 16
+ linear_units: 2048
+ n_layer: 24
+ dropout: 0
+ EOS: 1024
+ random_bert: 0
+inference:
+ top_k: 5
diff --git a/GPT_SoVITS/configs/s1mq.yaml b/GPT_SoVITS/configs/s1mq.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b554fd34f88faca8bc71eb3eaf6b1bdf301a326b
--- /dev/null
+++ b/GPT_SoVITS/configs/s1mq.yaml
@@ -0,0 +1,77 @@
+train:
+ seed: 1234
+ epochs: 100
+ batch_size: 6
+ gradient_accumulation: 4
+ save_every_n_epoch: 1
+ precision: 32
+ gradient_clip: 1.0
+optimizer:
+ lr: 0.01
+ lr_init: 0.00001
+ lr_end: 0.0001
+ warmup_steps: 2000
+ decay_steps: 40000
+data:
+ max_eval_sample: 8
+ max_sec: 40
+ num_workers: 1
+ pad_val: 1024 # same with EOS in model
+model:
+ saving_path: "ckpt/"
+ resume_checkpoint: null
+ vocoder_config_path: "quantizer/new_ckpt/config.json"
+ vocoder_ckpt_path: "quantizer/new_ckpt/g_00600000"
+ datadir: "/home/liweiche/GigaSpeech/wavs"
+ metapath: "/home/liweiche/GigaSpeech/train2.json"
+ val_metapath: "/home/liweiche/GigaSpeech/dev2.json"
+ sampledir: "logs/"
+ pretrained_path: null
+ lr: 0.0001
+ batch_size: 200.0
+ train_bucket_size: 8192
+ training_step: 800000
+ optim_flat_percent: 0.0
+ warmup_step: 50
+ adam_beta1: 0.9
+ adam_beta2: 0.98
+ ffd_size: 3072
+ hidden_size: 768
+ enc_nlayers: 6
+ dec_nlayers: 6
+ nheads: 12
+ ar_layer: 4
+ ar_ffd_size: 1024
+ ar_hidden_size: 256
+ ar_nheads: 4
+ aligner_softmax_temp: 1.0
+ layer_norm_eps: 0.00001
+ speaker_embed_dropout: 0.05
+ label_smoothing: 0.0
+ val_check_interval: 5000
+ check_val_every_n_epoch: 1
+ precision: "fp16"
+ nworkers: 16
+ distributed: true
+ accelerator: "ddp"
+ version: null
+ accumulate_grad_batches: 1
+ use_repetition_token: true
+ use_repetition_gating: false
+ repetition_penalty: 1.0
+ sampling_temperature: 1.0
+ top_k: -1
+ min_top_k: 3
+ top_p: 0.8
+ sample_num: 4
+ length_penalty_max_length: 15000
+ length_penalty_max_prob: 0.95
+ max_input_length: 2048
+ max_output_length: 2000
+ sample_rate: 16000
+ n_codes: 1024
+ n_cluster_groups: 1
+ phone_context_window: 4
+ phoneset_size: 1000
+inference:
+ top_k: 5
diff --git a/GPT_SoVITS/configs/s2.json b/GPT_SoVITS/configs/s2.json
new file mode 100644
index 0000000000000000000000000000000000000000..e44e1eb7d9864a20c76d35201f1ce0b9b15f3e95
--- /dev/null
+++ b/GPT_SoVITS/configs/s2.json
@@ -0,0 +1,90 @@
+{
+ "train": {
+ "log_interval": 100,
+ "eval_interval": 500,
+ "seed": 1234,
+ "epochs": 100,
+ "learning_rate": 0.0001,
+ "betas": [
+ 0.8,
+ 0.99
+ ],
+ "eps": 1e-09,
+ "batch_size": 32,
+ "fp16_run": true,
+ "lr_decay": 0.999875,
+ "segment_size": 20480,
+ "init_lr_ratio": 1,
+ "warmup_epochs": 0,
+ "c_mel": 45,
+ "c_kl": 1.0,
+ "text_low_lr_rate": 0.4
+ },
+ "data": {
+ "max_wav_value": 32768.0,
+ "sampling_rate": 32000,
+ "filter_length": 2048,
+ "hop_length": 640,
+ "win_length": 2048,
+ "n_mel_channels": 128,
+ "mel_fmin": 0.0,
+ "mel_fmax": null,
+ "add_blank": true,
+ "n_speakers": 300,
+ "cleaned_text": true
+ },
+ "model": {
+ "inter_channels": 192,
+ "hidden_channels": 192,
+ "filter_channels": 768,
+ "n_heads": 2,
+ "n_layers": 6,
+ "kernel_size": 3,
+ "p_dropout": 0.1,
+ "resblock": "1",
+ "resblock_kernel_sizes": [
+ 3,
+ 7,
+ 11
+ ],
+ "resblock_dilation_sizes": [
+ [
+ 1,
+ 3,
+ 5
+ ],
+ [
+ 1,
+ 3,
+ 5
+ ],
+ [
+ 1,
+ 3,
+ 5
+ ]
+ ],
+ "upsample_rates": [
+ 10,
+ 8,
+ 2,
+ 2,
+ 2
+ ],
+ "upsample_initial_channel": 512,
+ "upsample_kernel_sizes": [
+ 16,
+ 16,
+ 8,
+ 2,
+ 2
+ ],
+ "n_layers_q": 3,
+ "use_spectral_norm": false,
+ "gin_channels": 512,
+ "semantic_frame_rate": "25hz",
+ "freeze_quantizer": true
+ },
+ "s2_ckpt_dir": "logs/s2/big2k1",
+ "content_module": "cnhubert"
+}
\ No newline at end of file
diff --git a/GPT_SoVITS/configs/train.yaml b/GPT_SoVITS/configs/train.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..be5333571980807b7d8f788165035c02cb0d176c
--- /dev/null
+++ b/GPT_SoVITS/configs/train.yaml
@@ -0,0 +1,32 @@
+gpu:
+ n_card: 1
+ n_process_per_card: 2
+io:
+ text_path: D:\RVC1006\GPT-SoVITS\GPT_SoVITS
+ save_every_n_epoch: 1
+ precision: 16-mixed
+ gradient_clip: 1.0
+optimizer:
+ lr: 0.01
+ lr_init: 0.00001
+ lr_end: 0.0001
+ warmup_steps: 2000
+ decay_steps: 40000
+data:
+ max_eval_sample: 8
+ max_sec: 54
+ num_workers: 1
+ pad_val: 1024 # same with EOS in model
+model:
+ vocab_size: 1025
+ phoneme_vocab_size: 512
+ embedding_dim: 512
+ hidden_dim: 512
+ head: 16
+ linear_units: 2048
+ n_layer: 24
+ dropout: 0
+ EOS: 1024
+ random_bert: 0
+inference:
+ top_k: 5
diff --git a/GPT_SoVITS/feature_extractor/__init__.py b/GPT_SoVITS/feature_extractor/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..79aa9294fec8fd823afae7d1a18e3279533dd7cf
--- /dev/null
+++ b/GPT_SoVITS/feature_extractor/__init__.py
@@ -0,0 +1,6 @@
+from . import cnhubert, whisper_enc
+
+content_module_map = {
+ 'cnhubert': cnhubert,
+ 'whisper': whisper_enc
+}
\ No newline at end of file
diff --git a/GPT_SoVITS/feature_extractor/cnhubert.py b/GPT_SoVITS/feature_extractor/cnhubert.py
new file mode 100644
index 0000000000000000000000000000000000000000..dc155bddb8de0ddc4fec06def2c5a7746f25af4e
--- /dev/null
+++ b/GPT_SoVITS/feature_extractor/cnhubert.py
@@ -0,0 +1,104 @@
+import time
+
+import librosa
+import torch
+import torch.nn.functional as F
+import soundfile as sf
+import logging
+
+logging.getLogger("numba").setLevel(logging.WARNING)
+
+from transformers import (
+ Wav2Vec2FeatureExtractor,
+ HubertModel,
+)
+
+import utils
+import torch.nn as nn
+
+cnhubert_base_path = None
+
+
+class CNHubert(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.model = HubertModel.from_pretrained(cnhubert_base_path)
+ self.feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(
+ cnhubert_base_path
+ )
+
+ def forward(self, x):
+ input_values = self.feature_extractor(
+ x, return_tensors="pt", sampling_rate=16000
+ ).input_values.to(x.device)
+ feats = self.model(input_values)["last_hidden_state"]
+ return feats
+
+
+# class CNHubertLarge(nn.Module):
+# def __init__(self):
+# super().__init__()
+# self.model = HubertModel.from_pretrained("/data/docker/liujing04/gpt-vits/chinese-hubert-large")
+# self.feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained("/data/docker/liujing04/gpt-vits/chinese-hubert-large")
+# def forward(self, x):
+# input_values = self.feature_extractor(x, return_tensors="pt", sampling_rate=16000).input_values.to(x.device)
+# feats = self.model(input_values)["last_hidden_state"]
+# return feats
+#
+# class CVec(nn.Module):
+# def __init__(self):
+# super().__init__()
+# self.model = HubertModel.from_pretrained("/data/docker/liujing04/vc-webui-big/hubert_base")
+# self.feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained("/data/docker/liujing04/vc-webui-big/hubert_base")
+# def forward(self, x):
+# input_values = self.feature_extractor(x, return_tensors="pt", sampling_rate=16000).input_values.to(x.device)
+# feats = self.model(input_values)["last_hidden_state"]
+# return feats
+#
+# class cnw2v2base(nn.Module):
+# def __init__(self):
+# super().__init__()
+# self.model = Wav2Vec2Model.from_pretrained("/data/docker/liujing04/gpt-vits/chinese-wav2vec2-base")
+# self.feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained("/data/docker/liujing04/gpt-vits/chinese-wav2vec2-base")
+# def forward(self, x):
+# input_values = self.feature_extractor(x, return_tensors="pt", sampling_rate=16000).input_values.to(x.device)
+# feats = self.model(input_values)["last_hidden_state"]
+# return feats
+
+
+def get_model():
+ model = CNHubert()
+ model.eval()
+ return model
+
+
+# def get_large_model():
+# model = CNHubertLarge()
+# model.eval()
+# return model
+#
+# def get_model_cvec():
+# model = CVec()
+# model.eval()
+# return model
+#
+# def get_model_cnw2v2base():
+# model = cnw2v2base()
+# model.eval()
+# return model
+
+
+def get_content(hmodel, wav_16k_tensor):
+ with torch.no_grad():
+ feats = hmodel(wav_16k_tensor)
+ return feats.transpose(1, 2)
+
+
+if __name__ == "__main__":
+ model = get_model()
+ src_path = "/Users/Shared/原音频2.wav"
+ wav_16k_tensor = utils.load_wav_to_torch_and_resample(src_path, 16000)
+ model = model
+ wav_16k_tensor = wav_16k_tensor
+ feats = get_content(model, wav_16k_tensor)
+ print(feats.shape)
diff --git a/GPT_SoVITS/feature_extractor/whisper_enc.py b/GPT_SoVITS/feature_extractor/whisper_enc.py
new file mode 100644
index 0000000000000000000000000000000000000000..983c3e4d8a96232d29f41847abfedb70c42ebb02
--- /dev/null
+++ b/GPT_SoVITS/feature_extractor/whisper_enc.py
@@ -0,0 +1,25 @@
+import torch
+
+
+def get_model():
+ import whisper
+
+ model = whisper.load_model("small", device="cpu")
+
+ return model.encoder
+
+
+def get_content(model=None, wav_16k_tensor=None):
+ from whisper import log_mel_spectrogram, pad_or_trim
+
+ dev = next(model.parameters()).device
+ mel = log_mel_spectrogram(wav_16k_tensor).to(dev)[:, :3000]
+ # if torch.cuda.is_available():
+ # mel = mel.to(torch.float16)
+ feature_len = mel.shape[-1] // 2
+ assert mel.shape[-1] < 3000, "输入音频过长,只允许输入30以内音频"
+ with torch.no_grad():
+ feature = model(pad_or_trim(mel, 3000).unsqueeze(0))[
+ :1, :feature_len, :
+ ].transpose(1, 2)
+ return feature
diff --git a/GPT_SoVITS/inference_gui.py b/GPT_SoVITS/inference_gui.py
new file mode 100644
index 0000000000000000000000000000000000000000..f6cfdc5e23471ade2fd56dc16a5f101810219dee
--- /dev/null
+++ b/GPT_SoVITS/inference_gui.py
@@ -0,0 +1,340 @@
+import sys
+from PyQt5.QtCore import QEvent
+from PyQt5.QtWidgets import QApplication, QMainWindow, QLabel, QLineEdit, QPushButton, QTextEdit
+from PyQt5.QtWidgets import QGridLayout, QVBoxLayout, QWidget, QFileDialog, QStatusBar, QComboBox
+import soundfile as sf
+
+from tools.i18n.i18n import I18nAuto
+i18n = I18nAuto()
+
+from GPT_SoVITS.inference_webui import change_gpt_weights, change_sovits_weights, get_tts_wav
+
+
+class GPTSoVITSGUI(QMainWindow):
+ def __init__(self):
+ super().__init__()
+
+ self.init_ui()
+
+ def init_ui(self):
+ self.setWindowTitle('GPT-SoVITS GUI')
+ self.setGeometry(800, 450, 950, 850)
+
+ self.setStyleSheet("""
+ QWidget {
+ background-color: #a3d3b1;
+ }
+
+ QTabWidget::pane {
+ background-color: #a3d3b1;
+ }
+
+ QTabWidget::tab-bar {
+ alignment: left;
+ }
+
+ QTabBar::tab {
+ background: #8da4bf;
+ color: #ffffff;
+ padding: 8px;
+ }
+
+ QTabBar::tab:selected {
+ background: #2a3f54;
+ }
+
+ QLabel {
+ color: #000000;
+ }
+
+ QPushButton {
+ background-color: #4CAF50;
+ color: white;
+ padding: 8px;
+ border: 1px solid #4CAF50;
+ border-radius: 4px;
+ }
+
+ QPushButton:hover {
+ background-color: #45a049;
+ border: 1px solid #45a049;
+ box-shadow: 2px 2px 2px rgba(0, 0, 0, 0.1);
+ }
+ """)
+
+ license_text = (
+ "本软件以MIT协议开源, 作者不对软件具备任何控制力, 使用软件者、传播软件导出的声音者自负全责. "
+ "如不认可该条款, 则不能使用或引用软件包内任何代码和文件. 详见根目录LICENSE.")
+ license_label = QLabel(license_text)
+ license_label.setWordWrap(True)
+
+ self.GPT_model_label = QLabel("选择GPT模型:")
+ self.GPT_model_input = QLineEdit()
+ self.GPT_model_input.setPlaceholderText("拖拽或选择文件")
+ self.GPT_model_input.setReadOnly(True)
+ self.GPT_model_button = QPushButton("选择GPT模型文件")
+ self.GPT_model_button.clicked.connect(self.select_GPT_model)
+
+ self.SoVITS_model_label = QLabel("选择SoVITS模型:")
+ self.SoVITS_model_input = QLineEdit()
+ self.SoVITS_model_input.setPlaceholderText("拖拽或选择文件")
+ self.SoVITS_model_input.setReadOnly(True)
+ self.SoVITS_model_button = QPushButton("选择SoVITS模型文件")
+ self.SoVITS_model_button.clicked.connect(self.select_SoVITS_model)
+
+ self.ref_audio_label = QLabel("上传参考音频:")
+ self.ref_audio_input = QLineEdit()
+ self.ref_audio_input.setPlaceholderText("拖拽或选择文件")
+ self.ref_audio_input.setReadOnly(True)
+ self.ref_audio_button = QPushButton("选择音频文件")
+ self.ref_audio_button.clicked.connect(self.select_ref_audio)
+
+ self.ref_text_label = QLabel("参考音频文本:")
+ self.ref_text_input = QLineEdit()
+ self.ref_text_input.setPlaceholderText("拖拽或选择文件")
+ self.ref_text_input.setReadOnly(True)
+ self.ref_text_button = QPushButton("上传文本")
+ self.ref_text_button.clicked.connect(self.upload_ref_text)
+
+ self.language_label = QLabel("参考音频语言:")
+ self.language_combobox = QComboBox()
+ self.language_combobox.addItems(["中文", "英文", "日文"])
+
+ self.target_text_label = QLabel("合成目标文本:")
+ self.target_text_input = QLineEdit()
+ self.target_text_input.setPlaceholderText("拖拽或选择文件")
+ self.target_text_input.setReadOnly(True)
+ self.target_text_button = QPushButton("上传文本")
+ self.target_text_button.clicked.connect(self.upload_target_text)
+
+ self.language_label_02 = QLabel("合成音频语言:")
+ self.language_combobox_02 = QComboBox()
+ self.language_combobox_02.addItems(["中文", "英文", "日文"])
+
+ self.output_label = QLabel("输出音频路径:")
+ self.output_input = QLineEdit()
+ self.output_input.setPlaceholderText("拖拽或选择文件")
+ self.output_input.setReadOnly(True)
+ self.output_button = QPushButton("选择文件夹")
+ self.output_button.clicked.connect(self.select_output_path)
+
+ self.output_text = QTextEdit()
+ self.output_text.setReadOnly(True)
+
+ self.add_drag_drop_events([
+ self.GPT_model_input,
+ self.SoVITS_model_input,
+ self.ref_audio_input,
+ self.ref_text_input,
+ self.target_text_input,
+ self.output_input,
+ ])
+
+ self.synthesize_button = QPushButton("合成")
+ self.synthesize_button.clicked.connect(self.synthesize)
+
+ self.clear_output_button = QPushButton("清空输出")
+ self.clear_output_button.clicked.connect(self.clear_output)
+
+ self.status_bar = QStatusBar()
+
+ main_layout = QVBoxLayout()
+
+ input_layout = QGridLayout()
+ input_layout.setSpacing(10)
+
+ self.setLayout(input_layout)
+
+ input_layout.addWidget(license_label, 0, 0, 1, 3)
+
+ input_layout.addWidget(self.GPT_model_label, 1, 0)
+ input_layout.addWidget(self.GPT_model_input, 2, 0, 1, 2)
+ input_layout.addWidget(self.GPT_model_button, 2, 2)
+
+ input_layout.addWidget(self.SoVITS_model_label, 3, 0)
+ input_layout.addWidget(self.SoVITS_model_input, 4, 0, 1, 2)
+ input_layout.addWidget(self.SoVITS_model_button, 4, 2)
+
+ input_layout.addWidget(self.ref_audio_label, 5, 0)
+ input_layout.addWidget(self.ref_audio_input, 6, 0, 1, 2)
+ input_layout.addWidget(self.ref_audio_button, 6, 2)
+
+ input_layout.addWidget(self.language_label, 7, 0)
+ input_layout.addWidget(self.language_combobox, 8, 0, 1, 1)
+ input_layout.addWidget(self.ref_text_label, 9, 0)
+ input_layout.addWidget(self.ref_text_input, 10, 0, 1, 2)
+ input_layout.addWidget(self.ref_text_button, 10, 2)
+
+ input_layout.addWidget(self.language_label_02, 11, 0)
+ input_layout.addWidget(self.language_combobox_02, 12, 0, 1, 1)
+ input_layout.addWidget(self.target_text_label, 13, 0)
+ input_layout.addWidget(self.target_text_input, 14, 0, 1, 2)
+ input_layout.addWidget(self.target_text_button, 14, 2)
+
+ input_layout.addWidget(self.output_label, 15, 0)
+ input_layout.addWidget(self.output_input, 16, 0, 1, 2)
+ input_layout.addWidget(self.output_button, 16, 2)
+
+ main_layout.addLayout(input_layout)
+
+ output_layout = QVBoxLayout()
+ output_layout.addWidget(self.output_text)
+ main_layout.addLayout(output_layout)
+
+ main_layout.addWidget(self.synthesize_button)
+
+ main_layout.addWidget(self.clear_output_button)
+
+ main_layout.addWidget(self.status_bar)
+
+ self.central_widget = QWidget()
+ self.central_widget.setLayout(main_layout)
+ self.setCentralWidget(self.central_widget)
+
+ def dragEnterEvent(self, event):
+ if event.mimeData().hasUrls():
+ event.acceptProposedAction()
+
+ def dropEvent(self, event):
+ if event.mimeData().hasUrls():
+ file_paths = [url.toLocalFile() for url in event.mimeData().urls()]
+
+ if len(file_paths) == 1:
+ self.update_ref_audio(file_paths[0])
+ self.update_input_paths(self.ref_audio_input, file_paths[0])
+ else:
+ self.update_ref_audio(", ".join(file_paths))
+
+ def add_drag_drop_events(self, widgets):
+ for widget in widgets:
+ widget.setAcceptDrops(True)
+ widget.installEventFilter(self)
+
+ def eventFilter(self, obj, event):
+ if event.type() == QEvent.DragEnter:
+ mime_data = event.mimeData()
+ if mime_data.hasUrls():
+ event.acceptProposedAction()
+
+ elif event.type() == QEvent.Drop:
+ mime_data = event.mimeData()
+ if mime_data.hasUrls():
+ file_paths = [url.toLocalFile() for url in mime_data.urls()]
+ if len(file_paths) == 1:
+ self.update_input_paths(obj, file_paths[0])
+ else:
+ self.update_input_paths(obj, ", ".join(file_paths))
+ event.acceptProposedAction()
+
+ return super().eventFilter(obj, event)
+
+ def select_GPT_model(self):
+ file_path, _ = QFileDialog.getOpenFileName(self, "选择GPT模型文件", "", "GPT Files (*.ckpt)")
+ if file_path:
+ self.GPT_model_input.setText(file_path)
+
+ def select_SoVITS_model(self):
+ file_path, _ = QFileDialog.getOpenFileName(self, "选择SoVITS模型文件", "", "SoVITS Files (*.pth)")
+ if file_path:
+ self.SoVITS_model_input.setText(file_path)
+
+ def select_ref_audio(self):
+ options = QFileDialog.Options()
+ options |= QFileDialog.DontUseNativeDialog
+ options |= QFileDialog.ShowDirsOnly
+
+ file_dialog = QFileDialog()
+ file_dialog.setOptions(options)
+
+ file_dialog.setFileMode(QFileDialog.AnyFile)
+ file_dialog.setNameFilter("Audio Files (*.wav *.mp3)")
+
+ if file_dialog.exec_():
+ file_paths = file_dialog.selectedFiles()
+
+ if len(file_paths) == 1:
+ self.update_ref_audio(file_paths[0])
+ self.update_input_paths(self.ref_audio_input, file_paths[0])
+ else:
+ self.update_ref_audio(", ".join(file_paths))
+
+ def upload_ref_text(self):
+ file_path, _ = QFileDialog.getOpenFileName(self, "选择文本文件", "", "Text Files (*.txt)")
+ if file_path:
+ with open(file_path, 'r', encoding='utf-8') as file:
+ content = file.read()
+ self.ref_text_input.setText(content)
+ self.update_input_paths(self.ref_text_input, file_path)
+
+ def upload_target_text(self):
+ file_path, _ = QFileDialog.getOpenFileName(self, "选择文本文件", "", "Text Files (*.txt)")
+ if file_path:
+ with open(file_path, 'r', encoding='utf-8') as file:
+ content = file.read()
+ self.target_text_input.setText(content)
+ self.update_input_paths(self.target_text_input, file_path)
+
+ def select_output_path(self):
+ options = QFileDialog.Options()
+ options |= QFileDialog.DontUseNativeDialog
+ options |= QFileDialog.ShowDirsOnly
+
+ folder_dialog = QFileDialog()
+ folder_dialog.setOptions(options)
+ folder_dialog.setFileMode(QFileDialog.Directory)
+
+ if folder_dialog.exec_():
+ folder_path = folder_dialog.selectedFiles()[0]
+ self.output_input.setText(folder_path)
+
+ def update_ref_audio(self, file_path):
+ self.ref_audio_input.setText(file_path)
+
+ def update_input_paths(self, input_box, file_path):
+ input_box.setText(file_path)
+
+ def clear_output(self):
+ self.output_text.clear()
+
+ def synthesize(self):
+ GPT_model_path = self.GPT_model_input.text()
+ SoVITS_model_path = self.SoVITS_model_input.text()
+ ref_audio_path = self.ref_audio_input.text()
+ language_combobox = self.language_combobox.currentText()
+ language_combobox = i18n(language_combobox)
+ ref_text = self.ref_text_input.text()
+ language_combobox_02 = self.language_combobox_02.currentText()
+ language_combobox_02 = i18n(language_combobox_02)
+ target_text = self.target_text_input.text()
+ output_path = self.output_input.text()
+
+ change_gpt_weights(gpt_path=GPT_model_path)
+ change_sovits_weights(sovits_path=SoVITS_model_path)
+
+ synthesis_result = get_tts_wav(ref_wav_path=ref_audio_path,
+ prompt_text=ref_text,
+ prompt_language=language_combobox,
+ text=target_text,
+ text_language=language_combobox_02)
+
+ result_list = list(synthesis_result)
+
+ if result_list:
+ last_sampling_rate, last_audio_data = result_list[-1]
+ output_wav_path = os.path.join(output_path, "output.wav")
+ sf.write(output_wav_path, last_audio_data, last_sampling_rate)
+
+ result = "Audio saved to " + output_wav_path
+
+ self.status_bar.showMessage("合成完成!输出路径:" + output_wav_path, 5000)
+ self.output_text.append("处理结果:\n" + result)
+
+def main():
+ app = QApplication(sys.argv)
+ mainWin = GPTSoVITSGUI()
+ mainWin.show()
+ sys.exit(app.exec_())
+
+
+if __name__ == '__main__':
+ main()
diff --git a/GPT_SoVITS/inference_webui.py b/GPT_SoVITS/inference_webui.py
new file mode 100644
index 0000000000000000000000000000000000000000..269364721b2b68a1e4fb25338579e56a55be096f
--- /dev/null
+++ b/GPT_SoVITS/inference_webui.py
@@ -0,0 +1,618 @@
+'''
+按中英混合识别
+按日英混合识别
+多语种启动切分识别语种
+全部按中文识别
+全部按英文识别
+全部按日文识别
+'''
+import os, re, logging
+import LangSegment
+logging.getLogger("markdown_it").setLevel(logging.ERROR)
+logging.getLogger("urllib3").setLevel(logging.ERROR)
+logging.getLogger("httpcore").setLevel(logging.ERROR)
+logging.getLogger("httpx").setLevel(logging.ERROR)
+logging.getLogger("asyncio").setLevel(logging.ERROR)
+logging.getLogger("charset_normalizer").setLevel(logging.ERROR)
+logging.getLogger("torchaudio._extension").setLevel(logging.ERROR)
+import pdb
+import torch
+
+if os.path.exists("./gweight.txt"):
+ with open("./gweight.txt", 'r', encoding="utf-8") as file:
+ gweight_data = file.read()
+ gpt_path = os.environ.get(
+ "gpt_path", gweight_data)
+else:
+ gpt_path = os.environ.get(
+ "gpt_path", "GPT_SoVITS/pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt")
+
+if os.path.exists("./sweight.txt"):
+ with open("./sweight.txt", 'r', encoding="utf-8") as file:
+ sweight_data = file.read()
+ sovits_path = os.environ.get("sovits_path", sweight_data)
+else:
+ sovits_path = os.environ.get("sovits_path", "GPT_SoVITS/pretrained_models/s2G488k.pth")
+# gpt_path = os.environ.get(
+# "gpt_path", "pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt"
+# )
+# sovits_path = os.environ.get("sovits_path", "pretrained_models/s2G488k.pth")
+cnhubert_base_path = os.environ.get(
+ "cnhubert_base_path", "GPT_SoVITS/pretrained_models/chinese-hubert-base"
+)
+bert_path = os.environ.get(
+ "bert_path", "GPT_SoVITS/pretrained_models/chinese-roberta-wwm-ext-large"
+)
+infer_ttswebui = os.environ.get("infer_ttswebui", 9872)
+infer_ttswebui = int(infer_ttswebui)
+is_share = os.environ.get("is_share", "False")
+is_share = eval(is_share)
+if "_CUDA_VISIBLE_DEVICES" in os.environ:
+ os.environ["CUDA_VISIBLE_DEVICES"] = os.environ["_CUDA_VISIBLE_DEVICES"]
+is_half = eval(os.environ.get("is_half", "True")) and not torch.backends.mps.is_available()
+import gradio as gr
+from transformers import AutoModelForMaskedLM, AutoTokenizer
+import numpy as np
+import librosa
+from feature_extractor import cnhubert
+
+cnhubert.cnhubert_base_path = cnhubert_base_path
+
+from module.models import SynthesizerTrn
+from AR.models.t2s_lightning_module import Text2SemanticLightningModule
+from text import cleaned_text_to_sequence
+from text.cleaner import clean_text
+from time import time as ttime
+from module.mel_processing import spectrogram_torch
+from my_utils import load_audio
+from tools.i18n.i18n import I18nAuto
+
+i18n = I18nAuto()
+
+os.environ['PYTORCH_ENABLE_MPS_FALLBACK'] = '1' # 确保直接启动推理UI时也能够设置。
+
+if torch.cuda.is_available():
+ device = "cuda"
+else:
+ device = "cpu"
+
+tokenizer = AutoTokenizer.from_pretrained(bert_path)
+bert_model = AutoModelForMaskedLM.from_pretrained(bert_path)
+if is_half == True:
+ bert_model = bert_model.half().to(device)
+else:
+ bert_model = bert_model.to(device)
+
+
+def get_bert_feature(text, word2ph):
+ with torch.no_grad():
+ inputs = tokenizer(text, return_tensors="pt")
+ for i in inputs:
+ inputs[i] = inputs[i].to(device)
+ res = bert_model(**inputs, output_hidden_states=True)
+ res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()[1:-1]
+ assert len(word2ph) == len(text)
+ phone_level_feature = []
+ for i in range(len(word2ph)):
+ repeat_feature = res[i].repeat(word2ph[i], 1)
+ phone_level_feature.append(repeat_feature)
+ phone_level_feature = torch.cat(phone_level_feature, dim=0)
+ return phone_level_feature.T
+
+
+class DictToAttrRecursive(dict):
+ def __init__(self, input_dict):
+ super().__init__(input_dict)
+ for key, value in input_dict.items():
+ if isinstance(value, dict):
+ value = DictToAttrRecursive(value)
+ self[key] = value
+ setattr(self, key, value)
+
+ def __getattr__(self, item):
+ try:
+ return self[item]
+ except KeyError:
+ raise AttributeError(f"Attribute {item} not found")
+
+ def __setattr__(self, key, value):
+ if isinstance(value, dict):
+ value = DictToAttrRecursive(value)
+ super(DictToAttrRecursive, self).__setitem__(key, value)
+ super().__setattr__(key, value)
+
+ def __delattr__(self, item):
+ try:
+ del self[item]
+ except KeyError:
+ raise AttributeError(f"Attribute {item} not found")
+
+
+ssl_model = cnhubert.get_model()
+if is_half == True:
+ ssl_model = ssl_model.half().to(device)
+else:
+ ssl_model = ssl_model.to(device)
+
+
+def change_sovits_weights(sovits_path):
+ global vq_model, hps
+ dict_s2 = torch.load(sovits_path, map_location="cpu")
+ hps = dict_s2["config"]
+ hps = DictToAttrRecursive(hps)
+ hps.model.semantic_frame_rate = "25hz"
+ vq_model = SynthesizerTrn(
+ hps.data.filter_length // 2 + 1,
+ hps.train.segment_size // hps.data.hop_length,
+ n_speakers=hps.data.n_speakers,
+ **hps.model
+ )
+ if ("pretrained" not in sovits_path):
+ del vq_model.enc_q
+ if is_half == True:
+ vq_model = vq_model.half().to(device)
+ else:
+ vq_model = vq_model.to(device)
+ vq_model.eval()
+ print(vq_model.load_state_dict(dict_s2["weight"], strict=False))
+ with open("./sweight.txt", "w", encoding="utf-8") as f:
+ f.write(sovits_path)
+
+
+change_sovits_weights(sovits_path)
+
+
+def change_gpt_weights(gpt_path):
+ global hz, max_sec, t2s_model, config
+ hz = 50
+ dict_s1 = torch.load(gpt_path, map_location="cpu")
+ config = dict_s1["config"]
+ max_sec = config["data"]["max_sec"]
+ t2s_model = Text2SemanticLightningModule(config, "****", is_train=False)
+ t2s_model.load_state_dict(dict_s1["weight"])
+ if is_half == True:
+ t2s_model = t2s_model.half()
+ t2s_model = t2s_model.to(device)
+ t2s_model.eval()
+ total = sum([param.nelement() for param in t2s_model.parameters()])
+ print("Number of parameter: %.2fM" % (total / 1e6))
+ with open("./gweight.txt", "w", encoding="utf-8") as f: f.write(gpt_path)
+
+
+change_gpt_weights(gpt_path)
+
+
+def get_spepc(hps, filename):
+ audio = load_audio(filename, int(hps.data.sampling_rate))
+ audio = torch.FloatTensor(audio)
+ audio_norm = audio
+ audio_norm = audio_norm.unsqueeze(0)
+ spec = spectrogram_torch(
+ audio_norm,
+ hps.data.filter_length,
+ hps.data.sampling_rate,
+ hps.data.hop_length,
+ hps.data.win_length,
+ center=False,
+ )
+ return spec
+
+
+dict_language = {
+ i18n("中文"): "all_zh",#全部按中文识别
+ i18n("英文"): "en",#全部按英文识别#######不变
+ i18n("日文"): "all_ja",#全部按日文识别
+ i18n("中英混合"): "zh",#按中英混合识别####不变
+ i18n("日英混合"): "ja",#按日英混合识别####不变
+ i18n("多语种混合"): "auto",#多语种启动切分识别语种
+}
+
+
+def clean_text_inf(text, language):
+ phones, word2ph, norm_text = clean_text(text, language)
+ phones = cleaned_text_to_sequence(phones)
+ return phones, word2ph, norm_text
+
+dtype=torch.float16 if is_half == True else torch.float32
+def get_bert_inf(phones, word2ph, norm_text, language):
+ language=language.replace("all_","")
+ if language == "zh":
+ bert = get_bert_feature(norm_text, word2ph).to(device)#.to(dtype)
+ else:
+ bert = torch.zeros(
+ (1024, len(phones)),
+ dtype=torch.float16 if is_half == True else torch.float32,
+ ).to(device)
+
+ return bert
+
+
+splits = {",", "。", "?", "!", ",", ".", "?", "!", "~", ":", ":", "—", "…", }
+
+
+def get_first(text):
+ pattern = "[" + "".join(re.escape(sep) for sep in splits) + "]"
+ text = re.split(pattern, text)[0].strip()
+ return text
+
+
+def get_phones_and_bert(text,language):
+ if language in {"en","all_zh","all_ja"}:
+ language = language.replace("all_","")
+ if language == "en":
+ LangSegment.setfilters(["en"])
+ formattext = " ".join(tmp["text"] for tmp in LangSegment.getTexts(text))
+ else:
+ # 因无法区别中日文汉字,以用户输入为准
+ formattext = text
+ while " " in formattext:
+ formattext = formattext.replace(" ", " ")
+ phones, word2ph, norm_text = clean_text_inf(formattext, language)
+ if language == "zh":
+ bert = get_bert_feature(norm_text, word2ph).to(device)
+ else:
+ bert = torch.zeros(
+ (1024, len(phones)),
+ dtype=torch.float16 if is_half == True else torch.float32,
+ ).to(device)
+ elif language in {"zh", "ja","auto"}:
+ textlist=[]
+ langlist=[]
+ LangSegment.setfilters(["zh","ja","en"])
+ if language == "auto":
+ for tmp in LangSegment.getTexts(text):
+ langlist.append(tmp["lang"])
+ textlist.append(tmp["text"])
+ else:
+ for tmp in LangSegment.getTexts(text):
+ if tmp["lang"] == "en":
+ langlist.append(tmp["lang"])
+ else:
+ # 因无法区别中日文汉字,以用户输入为准
+ langlist.append(language)
+ textlist.append(tmp["text"])
+ print(textlist)
+ print(langlist)
+ phones_list = []
+ bert_list = []
+ norm_text_list = []
+ for i in range(len(textlist)):
+ lang = langlist[i]
+ phones, word2ph, norm_text = clean_text_inf(textlist[i], lang)
+ bert = get_bert_inf(phones, word2ph, norm_text, lang)
+ phones_list.append(phones)
+ norm_text_list.append(norm_text)
+ bert_list.append(bert)
+ bert = torch.cat(bert_list, dim=1)
+ phones = sum(phones_list, [])
+ norm_text = ''.join(norm_text_list)
+
+ return phones,bert.to(dtype),norm_text
+
+
+def merge_short_text_in_array(texts, threshold):
+ if (len(texts)) < 2:
+ return texts
+ result = []
+ text = ""
+ for ele in texts:
+ text += ele
+ if len(text) >= threshold:
+ result.append(text)
+ text = ""
+ if (len(text) > 0):
+ if len(result) == 0:
+ result.append(text)
+ else:
+ result[len(result) - 1] += text
+ return result
+
+def get_tts_wav(ref_wav_path, prompt_text, prompt_language, text, text_language, how_to_cut=i18n("不切"), top_k=20, top_p=0.6, temperature=0.6, ref_free = False):
+ if prompt_text is None or len(prompt_text) == 0:
+ ref_free = True
+ t0 = ttime()
+ prompt_language = dict_language[prompt_language]
+ text_language = dict_language[text_language]
+ if not ref_free:
+ prompt_text = prompt_text.strip("\n")
+ if (prompt_text[-1] not in splits): prompt_text += "。" if prompt_language != "en" else "."
+ print(i18n("实际输入的参考文本:"), prompt_text)
+ text = text.strip("\n")
+ if (text[0] not in splits and len(get_first(text)) < 4): text = "。" + text if text_language != "en" else "." + text
+
+ print(i18n("实际输入的目标文本:"), text)
+ zero_wav = np.zeros(
+ int(hps.data.sampling_rate * 0.3),
+ dtype=np.float16 if is_half == True else np.float32,
+ )
+ with torch.no_grad():
+ wav16k, sr = librosa.load(ref_wav_path, sr=16000)
+ if (wav16k.shape[0] > 160000 or wav16k.shape[0] < 48000):
+ raise OSError(i18n("参考音频在3~10秒范围外,请更换!"))
+ wav16k = torch.from_numpy(wav16k)
+ zero_wav_torch = torch.from_numpy(zero_wav)
+ if is_half == True:
+ wav16k = wav16k.half().to(device)
+ zero_wav_torch = zero_wav_torch.half().to(device)
+ else:
+ wav16k = wav16k.to(device)
+ zero_wav_torch = zero_wav_torch.to(device)
+ wav16k = torch.cat([wav16k, zero_wav_torch])
+ ssl_content = ssl_model.model(wav16k.unsqueeze(0))[
+ "last_hidden_state"
+ ].transpose(
+ 1, 2
+ ) # .float()
+ codes = vq_model.extract_latent(ssl_content)
+
+ prompt_semantic = codes[0, 0]
+ t1 = ttime()
+
+ if (how_to_cut == i18n("凑四句一切")):
+ text = cut1(text)
+ elif (how_to_cut == i18n("凑50字一切")):
+ text = cut2(text)
+ elif (how_to_cut == i18n("按中文句号。切")):
+ text = cut3(text)
+ elif (how_to_cut == i18n("按英文句号.切")):
+ text = cut4(text)
+ elif (how_to_cut == i18n("按标点符号切")):
+ text = cut5(text)
+ while "\n\n" in text:
+ text = text.replace("\n\n", "\n")
+ print(i18n("实际输入的目标文本(切句后):"), text)
+ texts = text.split("\n")
+ texts = merge_short_text_in_array(texts, 5)
+ audio_opt = []
+ if not ref_free:
+ phones1,bert1,norm_text1=get_phones_and_bert(prompt_text, prompt_language)
+
+ for text in texts:
+ # 解决输入目标文本的空行导致报错的问题
+ if (len(text.strip()) == 0):
+ continue
+ if (text[-1] not in splits): text += "。" if text_language != "en" else "."
+ print(i18n("实际输入的目标文本(每句):"), text)
+ phones2,bert2,norm_text2=get_phones_and_bert(text, text_language)
+ print(i18n("前端处理后的文本(每句):"), norm_text2)
+ if not ref_free:
+ bert = torch.cat([bert1, bert2], 1)
+ all_phoneme_ids = torch.LongTensor(phones1+phones2).to(device).unsqueeze(0)
+ else:
+ bert = bert2
+ all_phoneme_ids = torch.LongTensor(phones2).to(device).unsqueeze(0)
+
+ bert = bert.to(device).unsqueeze(0)
+ all_phoneme_len = torch.tensor([all_phoneme_ids.shape[-1]]).to(device)
+ prompt = prompt_semantic.unsqueeze(0).to(device)
+ t2 = ttime()
+ with torch.no_grad():
+ # pred_semantic = t2s_model.model.infer(
+ pred_semantic, idx = t2s_model.model.infer_panel(
+ all_phoneme_ids,
+ all_phoneme_len,
+ None if ref_free else prompt,
+ bert,
+ # prompt_phone_len=ph_offset,
+ top_k=top_k,
+ top_p=top_p,
+ temperature=temperature,
+ early_stop_num=hz * max_sec,
+ )
+ t3 = ttime()
+ # print(pred_semantic.shape,idx)
+ pred_semantic = pred_semantic[:, -idx:].unsqueeze(
+ 0
+ ) # .unsqueeze(0)#mq要多unsqueeze一次
+ refer = get_spepc(hps, ref_wav_path) # .to(device)
+ if is_half == True:
+ refer = refer.half().to(device)
+ else:
+ refer = refer.to(device)
+ # audio = vq_model.decode(pred_semantic, all_phoneme_ids, refer).detach().cpu().numpy()[0, 0]
+ audio = (
+ vq_model.decode(
+ pred_semantic, torch.LongTensor(phones2).to(device).unsqueeze(0), refer
+ )
+ .detach()
+ .cpu()
+ .numpy()[0, 0]
+ ) ###试试重建不带上prompt部分
+ max_audio=np.abs(audio).max()#简单防止16bit爆音
+ if max_audio>1:audio/=max_audio
+ audio_opt.append(audio)
+ audio_opt.append(zero_wav)
+ t4 = ttime()
+ print("%.3f\t%.3f\t%.3f\t%.3f" % (t1 - t0, t2 - t1, t3 - t2, t4 - t3))
+ yield hps.data.sampling_rate, (np.concatenate(audio_opt, 0) * 32768).astype(
+ np.int16
+ )
+
+
+def split(todo_text):
+ todo_text = todo_text.replace("……", "。").replace("——", ",")
+ if todo_text[-1] not in splits:
+ todo_text += "。"
+ i_split_head = i_split_tail = 0
+ len_text = len(todo_text)
+ todo_texts = []
+ while 1:
+ if i_split_head >= len_text:
+ break # 结尾一定有标点,所以直接跳出即可,最后一段在上次已加入
+ if todo_text[i_split_head] in splits:
+ i_split_head += 1
+ todo_texts.append(todo_text[i_split_tail:i_split_head])
+ i_split_tail = i_split_head
+ else:
+ i_split_head += 1
+ return todo_texts
+
+
+def cut1(inp):
+ inp = inp.strip("\n")
+ inps = split(inp)
+ split_idx = list(range(0, len(inps), 4))
+ split_idx[-1] = None
+ if len(split_idx) > 1:
+ opts = []
+ for idx in range(len(split_idx) - 1):
+ opts.append("".join(inps[split_idx[idx]: split_idx[idx + 1]]))
+ else:
+ opts = [inp]
+ return "\n".join(opts)
+
+
+def cut2(inp):
+ inp = inp.strip("\n")
+ inps = split(inp)
+ if len(inps) < 2:
+ return inp
+ opts = []
+ summ = 0
+ tmp_str = ""
+ for i in range(len(inps)):
+ summ += len(inps[i])
+ tmp_str += inps[i]
+ if summ > 50:
+ summ = 0
+ opts.append(tmp_str)
+ tmp_str = ""
+ if tmp_str != "":
+ opts.append(tmp_str)
+ # print(opts)
+ if len(opts) > 1 and len(opts[-1]) < 50: ##如果最后一个太短了,和前一个合一起
+ opts[-2] = opts[-2] + opts[-1]
+ opts = opts[:-1]
+ return "\n".join(opts)
+
+
+def cut3(inp):
+ inp = inp.strip("\n")
+ return "\n".join(["%s" % item for item in inp.strip("。").split("。")])
+
+
+def cut4(inp):
+ inp = inp.strip("\n")
+ return "\n".join(["%s" % item for item in inp.strip(".").split(".")])
+
+
+# contributed by https://github.com/AI-Hobbyist/GPT-SoVITS/blob/main/GPT_SoVITS/inference_webui.py
+def cut5(inp):
+ # if not re.search(r'[^\w\s]', inp[-1]):
+ # inp += '。'
+ inp = inp.strip("\n")
+ punds = r'[,.;?!、,。?!;:…]'
+ items = re.split(f'({punds})', inp)
+ mergeitems = ["".join(group) for group in zip(items[::2], items[1::2])]
+ # 在句子不存在符号或句尾无符号的时候保证文本完整
+ if len(items)%2 == 1:
+ mergeitems.append(items[-1])
+ opt = "\n".join(mergeitems)
+ return opt
+
+
+def custom_sort_key(s):
+ # 使用正则表达式提取字符串中的数字部分和非数字部分
+ parts = re.split('(\d+)', s)
+ # 将数字部分转换为整数,非数字部分保持不变
+ parts = [int(part) if part.isdigit() else part for part in parts]
+ return parts
+
+
+def change_choices():
+ SoVITS_names, GPT_names = get_weights_names()
+ return {"choices": sorted(SoVITS_names, key=custom_sort_key), "__type__": "update"}, {"choices": sorted(GPT_names, key=custom_sort_key), "__type__": "update"}
+
+
+pretrained_sovits_name = "GPT_SoVITS/pretrained_models/s2G488k.pth"
+pretrained_gpt_name = "GPT_SoVITS/pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt"
+SoVITS_weight_root = "SoVITS_weights"
+GPT_weight_root = "GPT_weights"
+os.makedirs(SoVITS_weight_root, exist_ok=True)
+os.makedirs(GPT_weight_root, exist_ok=True)
+
+
+def get_weights_names():
+ SoVITS_names = [pretrained_sovits_name]
+ for name in os.listdir(SoVITS_weight_root):
+ if name.endswith(".pth"): SoVITS_names.append("%s/%s" % (SoVITS_weight_root, name))
+ GPT_names = [pretrained_gpt_name]
+ for name in os.listdir(GPT_weight_root):
+ if name.endswith(".ckpt"): GPT_names.append("%s/%s" % (GPT_weight_root, name))
+ return SoVITS_names, GPT_names
+
+
+SoVITS_names, GPT_names = get_weights_names()
+
+with gr.Blocks(title="GPT-SoVITS WebUI") as app:
+ gr.Markdown(
+ value=i18n("本软件以MIT协议开源, 作者不对软件具备任何控制力, 使用软件者、传播软件导出的声音者自负全责.