mirror of
https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI.git
synced 2026-02-17 08:40:24 +00:00
chore(sync): merge dev into main (#1379)
* Optimize latency (#1259) * add attribute: configs/config.py Optimize latency: tools/rvc_for_realtime.py * new file: assets/Synthesizer_inputs.pth * fix: configs/config.py fix: tools/rvc_for_realtime.py * fix bug: infer/lib/infer_pack/models.py * new file: assets/hubert_inputs.pth new file: assets/rmvpe_inputs.pth modified: configs/config.py new features: infer/lib/rmvpe.py new features: tools/jit_export/__init__.py new features: tools/jit_export/get_hubert.py new features: tools/jit_export/get_rmvpe.py new features: tools/jit_export/get_synthesizer.py optimize: tools/rvc_for_realtime.py * optimize: tools/jit_export/get_synthesizer.py fix bug: tools/jit_export/__init__.py * Fixed a bug caused by using half on the CPU: infer/lib/rmvpe.py Fixed a bug caused by using half on the CPU: tools/jit_export/__init__.py Fixed CIRCULAR IMPORT: tools/jit_export/get_rmvpe.py Fixed CIRCULAR IMPORT: tools/jit_export/get_synthesizer.py Fixed a bug caused by using half on the CPU: tools/rvc_for_realtime.py * Remove useless code: infer/lib/rmvpe.py * Delete gui_v1 copy.py * Delete .vscode/launch.json * Delete jit_export_test.py * Delete tools/rvc_for_realtime copy.py * Delete configs/config.json * Delete .gitignore * Fix exceptions caused by switching inference devices: infer/lib/rmvpe.py Fix exceptions caused by switching inference devices: tools/jit_export/__init__.py Fix exceptions caused by switching inference devices: tools/rvc_for_realtime.py * restore * replace(you can undo this commit) * remove debug_print --------- Co-authored-by: Ftps <ftpsflandre@gmail.com> * Fixed some bugs when exporting ONNX model (#1254) * fix import (#1280) * fix import * lint * 🎨 同步 locale (#1242) Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> * Fix jit load and import issue (#1282) * fix jit model loading : infer/lib/rmvpe.py * modified: assets/hubert/.gitignore move file: assets/hubert_inputs.pth -> assets/hubert/hubert_inputs.pth modified: assets/rmvpe/.gitignore move file: assets/rmvpe_inputs.pth -> assets/rmvpe/rmvpe_inputs.pth fix import: gui_v1.py * feat(workflow): trigger on dev * feat(workflow): add close-pr on non-dev branch * Add input wav and delay time monitor for real-time gui (#1293) * feat(workflow): trigger on dev * feat(workflow): add close-pr on non-dev branch * 🎨 同步 locale (#1289) Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> * feat: edit PR template * add input wav and delay time monitor --------- Co-authored-by: 源文雨 <41315874+fumiama@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: RVC-Boss <129054828+RVC-Boss@users.noreply.github.com> * Optimize latency using scripted jit (#1291) * feat(workflow): trigger on dev * feat(workflow): add close-pr on non-dev branch * 🎨 同步 locale (#1289) Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> * feat: edit PR template * Optimize-latency-using-scripted: configs/config.py Optimize-latency-using-scripted: infer/lib/infer_pack/attentions.py Optimize-latency-using-scripted: infer/lib/infer_pack/commons.py Optimize-latency-using-scripted: infer/lib/infer_pack/models.py Optimize-latency-using-scripted: infer/lib/infer_pack/modules.py Optimize-latency-using-scripted: infer/lib/jit/__init__.py Optimize-latency-using-scripted: infer/lib/jit/get_hubert.py Optimize-latency-using-scripted: infer/lib/jit/get_rmvpe.py Optimize-latency-using-scripted: infer/lib/jit/get_synthesizer.py Optimize-latency-using-scripted: infer/lib/rmvpe.py Optimize-latency-using-scripted: tools/rvc_for_realtime.py * modified: infer/lib/infer_pack/models.py * fix some bug: configs/config.py fix some bug: infer/lib/infer_pack/models.py fix some bug: infer/lib/rmvpe.py * Fixed abnormal reference of logger in multiprocessing: infer/modules/train/train.py --------- Co-authored-by: 源文雨 <41315874+fumiama@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> * Format code (#1298) Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> * 🎨 同步 locale (#1299) Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> * feat: optimize actions * feat(workflow): add sync dev * feat: optimize actions * feat: optimize actions * feat: optimize actions * feat: optimize actions * feat: add jit options (#1303) Delete useless code: infer/lib/jit/get_synthesizer.py Optimized code: tools/rvc_for_realtime.py * Code refactor + re-design inference ui (#1304) * Code refacor + re-design inference ui * Fix tabname * i18n jp --------- Co-authored-by: Ftps <ftpsflandre@gmail.com> * feat: optimize actions * feat: optimize actions * Update README & en_US locale file (#1309) * critical: some bug fixes (#1322) * JIT acceleration switch does not support hot update * fix padding bug of rmvpe in torch-directml * fix padding bug of rmvpe in torch-directml * Fix STFT under torch_directml (#1330) * chore(format): run black on dev (#1318) Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> * chore(i18n): sync locale on dev (#1317) Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> * feat: allow for tta to be passed to uvr (#1361) * chore(format): run black on dev (#1373) Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> * Added script for automatically download all needed models at install (#1366) * Delete modules.py * Add files via upload * Add files via upload * Add files via upload * Add files via upload * chore(i18n): sync locale on dev (#1377) Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> * chore(format): run black on dev (#1376) Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> * Update IPEX library (#1362) * Update IPEX library * Update ipex index * chore(format): run black on dev (#1378) Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> --------- Co-authored-by: Chengjia Jiang <46401978+ChasonJiang@users.noreply.github.com> Co-authored-by: Ftps <ftpsflandre@gmail.com> Co-authored-by: shizuku_nia <102004222+ShizukuNia@users.noreply.github.com> Co-authored-by: Ftps <63702646+Tps-F@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: 源文雨 <41315874+fumiama@users.noreply.github.com> Co-authored-by: yxlllc <33565655+yxlllc@users.noreply.github.com> Co-authored-by: RVC-Boss <129054828+RVC-Boss@users.noreply.github.com> Co-authored-by: Blaise <133521603+blaise-tk@users.noreply.github.com> Co-authored-by: Rice Cake <gak141808@gmail.com> Co-authored-by: AWAS666 <33494149+AWAS666@users.noreply.github.com> Co-authored-by: Dmitry <nda2911@yandex.ru> Co-authored-by: Disty0 <47277141+Disty0@users.noreply.github.com>
This commit is contained in:
github-actions[bot]
committed by
GitHub
GitHub
parent
fe166e7f3d
commit
e9dd11bddb
163
infer/lib/jit/__init__.py
Normal file
163
infer/lib/jit/__init__.py
Normal file
@@ -0,0 +1,163 @@
|
||||
from io import BytesIO
|
||||
import pickle
|
||||
import time
|
||||
import torch
|
||||
from tqdm import tqdm
|
||||
from collections import OrderedDict
|
||||
|
||||
|
||||
def load_inputs(path, device, is_half=False):
|
||||
parm = torch.load(path, map_location=torch.device("cpu"))
|
||||
for key in parm.keys():
|
||||
parm[key] = parm[key].to(device)
|
||||
if is_half and parm[key].dtype == torch.float32:
|
||||
parm[key] = parm[key].half()
|
||||
elif not is_half and parm[key].dtype == torch.float16:
|
||||
parm[key] = parm[key].float()
|
||||
return parm
|
||||
|
||||
|
||||
def benchmark(
|
||||
model, inputs_path, device=torch.device("cpu"), epoch=1000, is_half=False
|
||||
):
|
||||
parm = load_inputs(inputs_path, device, is_half)
|
||||
total_ts = 0.0
|
||||
bar = tqdm(range(epoch))
|
||||
for i in bar:
|
||||
start_time = time.perf_counter()
|
||||
o = model(**parm)
|
||||
total_ts += time.perf_counter() - start_time
|
||||
print(f"num_epoch: {epoch} | avg time(ms): {(total_ts*1000)/epoch}")
|
||||
|
||||
|
||||
def jit_warm_up(model, inputs_path, device=torch.device("cpu"), epoch=5, is_half=False):
|
||||
benchmark(model, inputs_path, device, epoch=epoch, is_half=is_half)
|
||||
|
||||
|
||||
def to_jit_model(
|
||||
model_path,
|
||||
model_type: str,
|
||||
mode: str = "trace",
|
||||
inputs_path: str = None,
|
||||
device=torch.device("cpu"),
|
||||
is_half=False,
|
||||
):
|
||||
model = None
|
||||
if model_type.lower() == "synthesizer":
|
||||
from .get_synthesizer import get_synthesizer
|
||||
|
||||
model, _ = get_synthesizer(model_path, device)
|
||||
model.forward = model.infer
|
||||
elif model_type.lower() == "rmvpe":
|
||||
from .get_rmvpe import get_rmvpe
|
||||
|
||||
model = get_rmvpe(model_path, device)
|
||||
elif model_type.lower() == "hubert":
|
||||
from .get_hubert import get_hubert_model
|
||||
|
||||
model = get_hubert_model(model_path, device)
|
||||
model.forward = model.infer
|
||||
else:
|
||||
raise ValueError(f"No model type named {model_type}")
|
||||
model = model.eval()
|
||||
model = model.half() if is_half else model.float()
|
||||
if mode == "trace":
|
||||
assert not inputs_path
|
||||
inputs = load_inputs(inputs_path, device, is_half)
|
||||
model_jit = torch.jit.trace(model, example_kwarg_inputs=inputs)
|
||||
elif mode == "script":
|
||||
model_jit = torch.jit.script(model)
|
||||
model_jit.to(device)
|
||||
model_jit = model_jit.half() if is_half else model_jit.float()
|
||||
# model = model.half() if is_half else model.float()
|
||||
return (model, model_jit)
|
||||
|
||||
|
||||
def export(
|
||||
model: torch.nn.Module,
|
||||
mode: str = "trace",
|
||||
inputs: dict = None,
|
||||
device=torch.device("cpu"),
|
||||
is_half: bool = False,
|
||||
) -> dict:
|
||||
model = model.half() if is_half else model.float()
|
||||
model.eval()
|
||||
if mode == "trace":
|
||||
assert inputs is not None
|
||||
model_jit = torch.jit.trace(model, example_kwarg_inputs=inputs)
|
||||
elif mode == "script":
|
||||
model_jit = torch.jit.script(model)
|
||||
model_jit.to(device)
|
||||
model_jit = model_jit.half() if is_half else model_jit.float()
|
||||
buffer = BytesIO()
|
||||
# model_jit=model_jit.cpu()
|
||||
torch.jit.save(model_jit, buffer)
|
||||
del model_jit
|
||||
cpt = OrderedDict()
|
||||
cpt["model"] = buffer.getvalue()
|
||||
cpt["is_half"] = is_half
|
||||
return cpt
|
||||
|
||||
|
||||
def load(path: str):
|
||||
with open(path, "rb") as f:
|
||||
return pickle.load(f)
|
||||
|
||||
|
||||
def save(ckpt: dict, save_path: str):
|
||||
with open(save_path, "wb") as f:
|
||||
pickle.dump(ckpt, f)
|
||||
|
||||
|
||||
def rmvpe_jit_export(
|
||||
model_path: str,
|
||||
mode: str = "script",
|
||||
inputs_path: str = None,
|
||||
save_path: str = None,
|
||||
device=torch.device("cpu"),
|
||||
is_half=False,
|
||||
):
|
||||
if not save_path:
|
||||
save_path = model_path.rstrip(".pth")
|
||||
save_path += ".half.jit" if is_half else ".jit"
|
||||
if "cuda" in str(device) and ":" not in str(device):
|
||||
device = torch.device("cuda:0")
|
||||
from .get_rmvpe import get_rmvpe
|
||||
|
||||
model = get_rmvpe(model_path, device)
|
||||
inputs = None
|
||||
if mode == "trace":
|
||||
inputs = load_inputs(inputs_path, device, is_half)
|
||||
ckpt = export(model, mode, inputs, device, is_half)
|
||||
ckpt["device"] = str(device)
|
||||
save(ckpt, save_path)
|
||||
return ckpt
|
||||
|
||||
|
||||
def synthesizer_jit_export(
|
||||
model_path: str,
|
||||
mode: str = "script",
|
||||
inputs_path: str = None,
|
||||
save_path: str = None,
|
||||
device=torch.device("cpu"),
|
||||
is_half=False,
|
||||
):
|
||||
if not save_path:
|
||||
save_path = model_path.rstrip(".pth")
|
||||
save_path += ".half.jit" if is_half else ".jit"
|
||||
if "cuda" in str(device) and ":" not in str(device):
|
||||
device = torch.device("cuda:0")
|
||||
from .get_synthesizer import get_synthesizer
|
||||
|
||||
model, cpt = get_synthesizer(model_path, device)
|
||||
assert isinstance(cpt, dict)
|
||||
model.forward = model.infer
|
||||
inputs = None
|
||||
if mode == "trace":
|
||||
inputs = load_inputs(inputs_path, device, is_half)
|
||||
ckpt = export(model, mode, inputs, device, is_half)
|
||||
cpt.pop("weight")
|
||||
cpt["model"] = ckpt["model"]
|
||||
cpt["device"] = device
|
||||
save(cpt, save_path)
|
||||
return cpt
|
||||
342
infer/lib/jit/get_hubert.py
Normal file
342
infer/lib/jit/get_hubert.py
Normal file
@@ -0,0 +1,342 @@
|
||||
import math
|
||||
import random
|
||||
from typing import Optional, Tuple
|
||||
from fairseq.checkpoint_utils import load_model_ensemble_and_task
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
# from fairseq.data.data_utils import compute_mask_indices
|
||||
from fairseq.utils import index_put
|
||||
|
||||
|
||||
# @torch.jit.script
|
||||
def pad_to_multiple(x, multiple, dim=-1, value=0):
|
||||
# Inspired from https://github.com/lucidrains/local-attention/blob/master/local_attention/local_attention.py#L41
|
||||
if x is None:
|
||||
return None, 0
|
||||
tsz = x.size(dim)
|
||||
m = tsz / multiple
|
||||
remainder = math.ceil(m) * multiple - tsz
|
||||
if int(tsz % multiple) == 0:
|
||||
return x, 0
|
||||
pad_offset = (0,) * (-1 - dim) * 2
|
||||
|
||||
return F.pad(x, (*pad_offset, 0, remainder), value=value), remainder
|
||||
|
||||
|
||||
def extract_features(
|
||||
self,
|
||||
x,
|
||||
padding_mask=None,
|
||||
tgt_layer=None,
|
||||
min_layer=0,
|
||||
):
|
||||
if padding_mask is not None:
|
||||
x = index_put(x, padding_mask, 0)
|
||||
|
||||
x_conv = self.pos_conv(x.transpose(1, 2))
|
||||
x_conv = x_conv.transpose(1, 2)
|
||||
x = x + x_conv
|
||||
|
||||
if not self.layer_norm_first:
|
||||
x = self.layer_norm(x)
|
||||
|
||||
# pad to the sequence length dimension
|
||||
x, pad_length = pad_to_multiple(x, self.required_seq_len_multiple, dim=-2, value=0)
|
||||
if pad_length > 0 and padding_mask is None:
|
||||
padding_mask = x.new_zeros((x.size(0), x.size(1)), dtype=torch.bool)
|
||||
padding_mask[:, -pad_length:] = True
|
||||
else:
|
||||
padding_mask, _ = pad_to_multiple(
|
||||
padding_mask, self.required_seq_len_multiple, dim=-1, value=True
|
||||
)
|
||||
x = F.dropout(x, p=self.dropout, training=self.training)
|
||||
|
||||
# B x T x C -> T x B x C
|
||||
x = x.transpose(0, 1)
|
||||
|
||||
layer_results = []
|
||||
r = None
|
||||
for i, layer in enumerate(self.layers):
|
||||
dropout_probability = np.random.random() if self.layerdrop > 0 else 1
|
||||
if not self.training or (dropout_probability > self.layerdrop):
|
||||
x, (z, lr) = layer(
|
||||
x, self_attn_padding_mask=padding_mask, need_weights=False
|
||||
)
|
||||
if i >= min_layer:
|
||||
layer_results.append((x, z, lr))
|
||||
if i == tgt_layer:
|
||||
r = x
|
||||
break
|
||||
|
||||
if r is not None:
|
||||
x = r
|
||||
|
||||
# T x B x C -> B x T x C
|
||||
x = x.transpose(0, 1)
|
||||
|
||||
# undo paddding
|
||||
if pad_length > 0:
|
||||
x = x[:, :-pad_length]
|
||||
|
||||
def undo_pad(a, b, c):
|
||||
return (
|
||||
a[:-pad_length],
|
||||
b[:-pad_length] if b is not None else b,
|
||||
c[:-pad_length],
|
||||
)
|
||||
|
||||
layer_results = [undo_pad(*u) for u in layer_results]
|
||||
|
||||
return x, layer_results
|
||||
|
||||
|
||||
def compute_mask_indices(
|
||||
shape: Tuple[int, int],
|
||||
padding_mask: Optional[torch.Tensor],
|
||||
mask_prob: float,
|
||||
mask_length: int,
|
||||
mask_type: str = "static",
|
||||
mask_other: float = 0.0,
|
||||
min_masks: int = 0,
|
||||
no_overlap: bool = False,
|
||||
min_space: int = 0,
|
||||
require_same_masks: bool = True,
|
||||
mask_dropout: float = 0.0,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Computes random mask spans for a given shape
|
||||
|
||||
Args:
|
||||
shape: the the shape for which to compute masks.
|
||||
should be of size 2 where first element is batch size and 2nd is timesteps
|
||||
padding_mask: optional padding mask of the same size as shape, which will prevent masking padded elements
|
||||
mask_prob: probability for each token to be chosen as start of the span to be masked. this will be multiplied by
|
||||
number of timesteps divided by length of mask span to mask approximately this percentage of all elements.
|
||||
however due to overlaps, the actual number will be smaller (unless no_overlap is True)
|
||||
mask_type: how to compute mask lengths
|
||||
static = fixed size
|
||||
uniform = sample from uniform distribution [mask_other, mask_length*2]
|
||||
normal = sample from normal distribution with mean mask_length and stdev mask_other. mask is min 1 element
|
||||
poisson = sample from possion distribution with lambda = mask length
|
||||
min_masks: minimum number of masked spans
|
||||
no_overlap: if false, will switch to an alternative recursive algorithm that prevents spans from overlapping
|
||||
min_space: only used if no_overlap is True, this is how many elements to keep unmasked between spans
|
||||
require_same_masks: if true, will randomly drop out masks until same amount of masks remains in each sample
|
||||
mask_dropout: randomly dropout this percentage of masks in each example
|
||||
"""
|
||||
|
||||
bsz, all_sz = shape
|
||||
mask = torch.full((bsz, all_sz), False)
|
||||
|
||||
all_num_mask = int(
|
||||
# add a random number for probabilistic rounding
|
||||
mask_prob * all_sz / float(mask_length)
|
||||
+ torch.rand([1]).item()
|
||||
)
|
||||
|
||||
all_num_mask = max(min_masks, all_num_mask)
|
||||
|
||||
mask_idcs = []
|
||||
for i in range(bsz):
|
||||
if padding_mask is not None:
|
||||
sz = all_sz - padding_mask[i].long().sum().item()
|
||||
num_mask = int(mask_prob * sz / float(mask_length) + np.random.rand())
|
||||
num_mask = max(min_masks, num_mask)
|
||||
else:
|
||||
sz = all_sz
|
||||
num_mask = all_num_mask
|
||||
|
||||
if mask_type == "static":
|
||||
lengths = torch.full([num_mask], mask_length)
|
||||
elif mask_type == "uniform":
|
||||
lengths = torch.randint(mask_other, mask_length * 2 + 1, size=[num_mask])
|
||||
elif mask_type == "normal":
|
||||
lengths = torch.normal(mask_length, mask_other, size=[num_mask])
|
||||
lengths = [max(1, int(round(x))) for x in lengths]
|
||||
else:
|
||||
raise Exception("unknown mask selection " + mask_type)
|
||||
|
||||
if sum(lengths) == 0:
|
||||
lengths[0] = min(mask_length, sz - 1)
|
||||
|
||||
if no_overlap:
|
||||
mask_idc = []
|
||||
|
||||
def arrange(s, e, length, keep_length):
|
||||
span_start = torch.randint(low=s, high=e - length, size=[1]).item()
|
||||
mask_idc.extend(span_start + i for i in range(length))
|
||||
|
||||
new_parts = []
|
||||
if span_start - s - min_space >= keep_length:
|
||||
new_parts.append((s, span_start - min_space + 1))
|
||||
if e - span_start - length - min_space > keep_length:
|
||||
new_parts.append((span_start + length + min_space, e))
|
||||
return new_parts
|
||||
|
||||
parts = [(0, sz)]
|
||||
min_length = min(lengths)
|
||||
for length in sorted(lengths, reverse=True):
|
||||
t = [e - s if e - s >= length + min_space else 0 for s, e in parts]
|
||||
lens = torch.asarray(t, dtype=torch.int)
|
||||
l_sum = torch.sum(lens)
|
||||
if l_sum == 0:
|
||||
break
|
||||
probs = lens / torch.sum(lens)
|
||||
c = torch.multinomial(probs.float(), len(parts)).item()
|
||||
s, e = parts.pop(c)
|
||||
parts.extend(arrange(s, e, length, min_length))
|
||||
mask_idc = torch.asarray(mask_idc)
|
||||
else:
|
||||
min_len = min(lengths)
|
||||
if sz - min_len <= num_mask:
|
||||
min_len = sz - num_mask - 1
|
||||
mask_idc = torch.asarray(
|
||||
random.sample([i for i in range(sz - min_len)], num_mask)
|
||||
)
|
||||
mask_idc = torch.asarray(
|
||||
[
|
||||
mask_idc[j] + offset
|
||||
for j in range(len(mask_idc))
|
||||
for offset in range(lengths[j])
|
||||
]
|
||||
)
|
||||
|
||||
mask_idcs.append(torch.unique(mask_idc[mask_idc < sz]))
|
||||
|
||||
min_len = min([len(m) for m in mask_idcs])
|
||||
for i, mask_idc in enumerate(mask_idcs):
|
||||
if isinstance(mask_idc, torch.Tensor):
|
||||
mask_idc = torch.asarray(mask_idc, dtype=torch.float)
|
||||
if len(mask_idc) > min_len and require_same_masks:
|
||||
mask_idc = torch.asarray(
|
||||
random.sample([i for i in range(mask_idc)], min_len)
|
||||
)
|
||||
if mask_dropout > 0:
|
||||
num_holes = int(round(len(mask_idc) * mask_dropout))
|
||||
mask_idc = torch.asarray(
|
||||
random.sample([i for i in range(mask_idc)], len(mask_idc) - num_holes)
|
||||
)
|
||||
|
||||
mask[i, mask_idc.int()] = True
|
||||
|
||||
return mask
|
||||
|
||||
|
||||
def apply_mask(self, x, padding_mask, target_list):
|
||||
B, T, C = x.shape
|
||||
torch.zeros_like(x)
|
||||
if self.mask_prob > 0:
|
||||
mask_indices = compute_mask_indices(
|
||||
(B, T),
|
||||
padding_mask,
|
||||
self.mask_prob,
|
||||
self.mask_length,
|
||||
self.mask_selection,
|
||||
self.mask_other,
|
||||
min_masks=2,
|
||||
no_overlap=self.no_mask_overlap,
|
||||
min_space=self.mask_min_space,
|
||||
)
|
||||
mask_indices = mask_indices.to(x.device)
|
||||
x[mask_indices] = self.mask_emb
|
||||
else:
|
||||
mask_indices = None
|
||||
|
||||
if self.mask_channel_prob > 0:
|
||||
mask_channel_indices = compute_mask_indices(
|
||||
(B, C),
|
||||
None,
|
||||
self.mask_channel_prob,
|
||||
self.mask_channel_length,
|
||||
self.mask_channel_selection,
|
||||
self.mask_channel_other,
|
||||
no_overlap=self.no_mask_channel_overlap,
|
||||
min_space=self.mask_channel_min_space,
|
||||
)
|
||||
mask_channel_indices = (
|
||||
mask_channel_indices.to(x.device).unsqueeze(1).expand(-1, T, -1)
|
||||
)
|
||||
x[mask_channel_indices] = 0
|
||||
|
||||
return x, mask_indices
|
||||
|
||||
|
||||
def get_hubert_model(
|
||||
model_path="assets/hubert/hubert_base.pt", device=torch.device("cpu")
|
||||
):
|
||||
models, _, _ = load_model_ensemble_and_task(
|
||||
[model_path],
|
||||
suffix="",
|
||||
)
|
||||
hubert_model = models[0]
|
||||
hubert_model = hubert_model.to(device)
|
||||
|
||||
def _apply_mask(x, padding_mask, target_list):
|
||||
return apply_mask(hubert_model, x, padding_mask, target_list)
|
||||
|
||||
hubert_model.apply_mask = _apply_mask
|
||||
|
||||
def _extract_features(
|
||||
x,
|
||||
padding_mask=None,
|
||||
tgt_layer=None,
|
||||
min_layer=0,
|
||||
):
|
||||
return extract_features(
|
||||
hubert_model.encoder,
|
||||
x,
|
||||
padding_mask=padding_mask,
|
||||
tgt_layer=tgt_layer,
|
||||
min_layer=min_layer,
|
||||
)
|
||||
|
||||
hubert_model.encoder.extract_features = _extract_features
|
||||
|
||||
hubert_model._forward = hubert_model.forward
|
||||
|
||||
def hubert_extract_features(
|
||||
self,
|
||||
source: torch.Tensor,
|
||||
padding_mask: Optional[torch.Tensor] = None,
|
||||
mask: bool = False,
|
||||
ret_conv: bool = False,
|
||||
output_layer: Optional[int] = None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
res = self._forward(
|
||||
source,
|
||||
padding_mask=padding_mask,
|
||||
mask=mask,
|
||||
features_only=True,
|
||||
output_layer=output_layer,
|
||||
)
|
||||
feature = res["features"] if ret_conv else res["x"]
|
||||
return feature, res["padding_mask"]
|
||||
|
||||
def _hubert_extract_features(
|
||||
source: torch.Tensor,
|
||||
padding_mask: Optional[torch.Tensor] = None,
|
||||
mask: bool = False,
|
||||
ret_conv: bool = False,
|
||||
output_layer: Optional[int] = None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
return hubert_extract_features(
|
||||
hubert_model, source, padding_mask, mask, ret_conv, output_layer
|
||||
)
|
||||
|
||||
hubert_model.extract_features = _hubert_extract_features
|
||||
|
||||
def infer(source, padding_mask, output_layer: torch.Tensor):
|
||||
output_layer = output_layer.item()
|
||||
logits = hubert_model.extract_features(
|
||||
source=source, padding_mask=padding_mask, output_layer=output_layer
|
||||
)
|
||||
feats = hubert_model.final_proj(logits[0]) if output_layer == 9 else logits[0]
|
||||
return feats
|
||||
|
||||
hubert_model.infer = infer
|
||||
# hubert_model.forward=infer
|
||||
# hubert_model.forward
|
||||
|
||||
return hubert_model
|
||||
12
infer/lib/jit/get_rmvpe.py
Normal file
12
infer/lib/jit/get_rmvpe.py
Normal file
@@ -0,0 +1,12 @@
|
||||
import torch
|
||||
|
||||
|
||||
def get_rmvpe(model_path="assets/rmvpe/rmvpe.pt", device=torch.device("cpu")):
|
||||
from infer.lib.rmvpe import E2E
|
||||
|
||||
model = E2E(4, 1, (2, 2))
|
||||
ckpt = torch.load(model_path, map_location=device)
|
||||
model.load_state_dict(ckpt)
|
||||
model.eval()
|
||||
model = model.to(device)
|
||||
return model
|
||||
37
infer/lib/jit/get_synthesizer.py
Normal file
37
infer/lib/jit/get_synthesizer.py
Normal file
@@ -0,0 +1,37 @@
|
||||
import torch
|
||||
|
||||
|
||||
def get_synthesizer(pth_path, device=torch.device("cpu")):
|
||||
from infer.lib.infer_pack.models import (
|
||||
SynthesizerTrnMs256NSFsid,
|
||||
SynthesizerTrnMs256NSFsid_nono,
|
||||
SynthesizerTrnMs768NSFsid,
|
||||
SynthesizerTrnMs768NSFsid_nono,
|
||||
)
|
||||
|
||||
cpt = torch.load(pth_path, map_location=torch.device("cpu"))
|
||||
# tgt_sr = cpt["config"][-1]
|
||||
cpt["config"][-3] = cpt["weight"]["emb_g.weight"].shape[0]
|
||||
if_f0 = cpt.get("f0", 1)
|
||||
version = cpt.get("version", "v1")
|
||||
if version == "v1":
|
||||
if if_f0 == 1:
|
||||
net_g = SynthesizerTrnMs256NSFsid(*cpt["config"], is_half=False)
|
||||
else:
|
||||
net_g = SynthesizerTrnMs256NSFsid_nono(*cpt["config"])
|
||||
elif version == "v2":
|
||||
if if_f0 == 1:
|
||||
net_g = SynthesizerTrnMs768NSFsid(*cpt["config"], is_half=False)
|
||||
else:
|
||||
net_g = SynthesizerTrnMs768NSFsid_nono(*cpt["config"])
|
||||
del net_g.enc_q
|
||||
# net_g.forward = net_g.infer
|
||||
# ckpt = {}
|
||||
# ckpt["config"] = cpt["config"]
|
||||
# ckpt["f0"] = if_f0
|
||||
# ckpt["version"] = version
|
||||
# ckpt["info"] = cpt.get("info", "0epoch")
|
||||
net_g.load_state_dict(cpt["weight"], strict=False)
|
||||
net_g = net_g.float()
|
||||
net_g.eval().to(device)
|
||||
return net_g, cpt
|
||||
Reference in New Issue
Block a user