voice-changer/server/voice_changer/SoVitsSvc40/SoVitsSvc40.py

import sys
import os
from data.ModelSlot import SoVitsSvc40ModelSlot
from voice_changer.VoiceChangerParamsManager import VoiceChangerParamsManager

from voice_changer.utils.VoiceChangerModel import AudioInOut, VoiceChangerModel
from voice_changer.utils.VoiceChangerParams import VoiceChangerParams

if sys.platform.startswith("darwin"):
    baseDir = [x for x in sys.path if x.endswith("Contents/MacOS")]
    if len(baseDir) != 1:
        print("baseDir should be only one ", baseDir)
        sys.exit()
    modulePath = os.path.join(baseDir[0], "so-vits-svc-40")
    sys.path.append(modulePath)
else:
    sys.path.append("so-vits-svc-40")

from dataclasses import dataclass, asdict, field
import numpy as np
import torch
import onnxruntime

# onnxruntime.set_default_logger_severity(3)

import pyworld as pw

# from models import SynthesizerTrn  # type:ignore
from .models.models import SynthesizerTrn
from .models.utils import (
    interpolate_f0,
    get_hparams_from_file,
    load_checkpoint,
    repeat_expand_2d,
    get_hubert_content,
)
from .models.cluster import get_cluster_model, get_cluster_center_result
from fairseq import checkpoint_utils
import librosa

from Exceptions import NoModeLoadedException


providers = [
    "OpenVINOExecutionProvider",
    "CUDAExecutionProvider",
    "DmlExecutionProvider",
    "CPUExecutionProvider",
]


@dataclass
class SoVitsSvc40Settings:
    gpu: int = -9999
    dstId: int = 0

    f0Detector: str = "harvest"  # dio or harvest
    tran: int = 20
    noiseScale: float = 0.3
    predictF0: int = 0  # 0:False, 1:True
    silentThreshold: float = 0.00001
    extraConvertSize: int = 1024 * 32
    clusterInferRatio: float = 0.1

    speakers: dict[str, int] = field(default_factory=lambda: {})

    # ↓mutableな物だけ列挙
    intData = ["gpu", "dstId", "tran", "predictF0"]
    floatData = ["noiseScale", "silentThreshold", "clusterInferRatio"]
    strData = ["f0Detector"]


class SoVitsSvc40(VoiceChangerModel):
    def __init__(self, params: VoiceChangerParams, slotInfo: SoVitsSvc40ModelSlot):
        print("[Voice Changer] [so-vits-svc40] Creating instance ")
        self.voiceChangerType = "so-vits-svc-40"
        self.settings = SoVitsSvc40Settings()
        self.net_g = None
        self.onnx_session = None

        self.params = params

        # hubert model
        try:
            models, saved_cfg, task = checkpoint_utils.load_model_ensemble_and_task(
                [self.params.hubert_base],
                suffix="",
            )
            model = models[0]
            model.eval()
            self.hubert_model = model.cpu()
        except Exception as e:
            print("EXCEPTION during loading hubert/contentvec model", e)

        self.gpu_num = torch.cuda.device_count()
        self.audio_buffer: AudioInOut | None = None
        self.prevVol = 0
        self.slotInfo = slotInfo
        self.initialize()

    def initialize(self):
        print("[Voice Changer] [so-vits-svc40] Initializing... ")
        vcparams = VoiceChangerParamsManager.get_instance().params
        configPath = os.path.join(
            vcparams.model_dir, str(self.slotInfo.slotIndex), self.slotInfo.configFile
        )
        modelPath = os.path.join(
            vcparams.model_dir, str(self.slotInfo.slotIndex), self.slotInfo.modelFile
        )
        self.hps = get_hparams_from_file(configPath)
        self.settings.speakers = self.hps.spk

        # cluster
        try:
            if self.slotInfo.clusterFile is not None:
                clusterPath = os.path.join(
                    vcparams.model_dir,
                    str(self.slotInfo.slotIndex),
                    self.slotInfo.clusterFile,
                )
                self.cluster_model = get_cluster_model(clusterPath)
            else:
                self.cluster_model = None
        except Exception as e:
            print(
                "[Voice Changer] [so-vits-svc40] EXCEPTION during loading cluster model ",
                e,
            )
            print("[Voice Changer] [so-vits-svc40] fallback to without cluster")
            self.cluster_model = None

        # model
        if self.slotInfo.isONNX:
            providers, options = self.getOnnxExecutionProvider()
            self.onnx_session = onnxruntime.InferenceSession(
                modelPath,
                providers=providers,
                provider_options=options,
            )
        else:
            net_g = SynthesizerTrn(
                self.hps.data.filter_length // 2 + 1,
                self.hps.train.segment_size // self.hps.data.hop_length,
                **self.hps.model,
            )
            net_g.eval()
            self.net_g = net_g
            load_checkpoint(modelPath, self.net_g, None)

    def getOnnxExecutionProvider(self):
        availableProviders = onnxruntime.get_available_providers()
        devNum = torch.cuda.device_count()
        if (
            self.settings.gpu >= 0
            and "CUDAExecutionProvider" in availableProviders
            and devNum > 0
        ):
            return ["CUDAExecutionProvider"], [{"device_id": self.settings.gpu}]
        elif self.settings.gpu >= 0 and "DmlExecutionProvider" in availableProviders:
            return ["DmlExecutionProvider"], [{}]
        else:
            return ["CPUExecutionProvider"], [
                {
                    "intra_op_num_threads": 8,
                    "execution_mode": onnxruntime.ExecutionMode.ORT_PARALLEL,
                    "inter_op_num_threads": 8,
                }
            ]

    def update_settings(self, key: str, val: int | float | str):
        if key in self.settings.intData:
            val = int(val)
            setattr(self.settings, key, val)

            if key == "gpu" and self.slotInfo.isONNX:
                providers, options = self.getOnnxExecutionProvider()
                if self.onnx_session is not None:
                    self.onnx_session.set_providers(
                        providers=providers,
                        provider_options=options,
                    )

        elif key in self.settings.floatData:
            setattr(self.settings, key, float(val))
        elif key in self.settings.strData:
            setattr(self.settings, key, str(val))
        else:
            return False

        return True

    def get_info(self):
        data = asdict(self.settings)

        data["onnxExecutionProviders"] = (
            self.onnx_session.get_providers() if self.onnx_session is not None else []
        )

        return data

    def get_processing_sampling_rate(self):
        if hasattr(self, "hps") is False:
            raise NoModeLoadedException("config")
        return self.hps.data.sampling_rate

    def get_unit_f0(self, audio_buffer, tran):
        wav_44k = audio_buffer

        if self.settings.f0Detector == "dio":
            f0 = compute_f0_dio(
                wav_44k,
                sampling_rate=self.hps.data.sampling_rate,
                hop_length=self.hps.data.hop_length,
            )
        else:
            f0 = compute_f0_harvest(
                wav_44k,
                sampling_rate=self.hps.data.sampling_rate,
                hop_length=self.hps.data.hop_length,
            )

        if wav_44k.shape[0] % self.hps.data.hop_length != 0:
            print(
                f" !!! !!! !!! wav size not multiple of hopsize: {wav_44k.shape[0] / self.hps.data.hop_length}"
            )

        f0, uv = interpolate_f0(f0)
        f0 = torch.FloatTensor(f0)
        uv = torch.FloatTensor(uv)
        f0 = f0 * 2 ** (tran / 12)
        f0 = f0.unsqueeze(0)
        uv = uv.unsqueeze(0)

        wav16k_numpy = librosa.resample(
            audio_buffer, orig_sr=self.hps.data.sampling_rate, target_sr=16000
        )
        wav16k_tensor = torch.from_numpy(wav16k_numpy)

        if (self.settings.gpu < 0 or self.gpu_num == 0) or self.slotInfo.isONNX:
            dev = torch.device("cpu")
        else:
            dev = torch.device("cuda", index=self.settings.gpu)

        if hasattr(self, "content_vec_onnx"):
            c = self.content_vec_onnx.run(
                ["units"],
                {
                    "audio": wav16k_numpy.reshape(1, -1),
                },
            )
            c = torch.from_numpy(np.array(c)).squeeze(0).transpose(1, 2)
            # print("onnx hubert:", self.content_vec_onnx.get_providers())
        else:
            if self.hps.model.ssl_dim == 768:
                self.hubert_model = self.hubert_model.to(dev)
                wav16k_tensor = wav16k_tensor.to(dev)
                c = get_hubert_content_layer9(
                    self.hubert_model, wav_16k_tensor=wav16k_tensor
                )
            else:
                self.hubert_model = self.hubert_model.to(dev)
                wav16k_tensor = wav16k_tensor.to(dev)
                c = get_hubert_content(self.hubert_model, wav_16k_tensor=wav16k_tensor)

        uv = uv.to(dev)
        f0 = f0.to(dev)

        c = repeat_expand_2d(c.squeeze(0), f0.shape[1])

        if (
            self.settings.clusterInferRatio != 0
            and hasattr(self, "cluster_model")
            and self.cluster_model is not None
        ):
            speaker = [
                key
                for key, value in self.settings.speakers.items()
                if value == self.settings.dstId
            ]
            if len(speaker) != 1:
                pass
                # print("not only one speaker found.", speaker)
            else:
                cluster_c = get_cluster_center_result(
                    self.cluster_model, c.cpu().numpy().T, speaker[0]
                ).T
                cluster_c = torch.FloatTensor(cluster_c).to(dev)
                c = c.to(dev)
                c = (
                    self.settings.clusterInferRatio * cluster_c
                    + (1 - self.settings.clusterInferRatio) * c
                )

        c = c.unsqueeze(0)
        return c, f0, uv

    def generate_input(
        self,
        newData: AudioInOut,
        inputSize: int,
        crossfadeSize: int,
        solaSearchFrame: int = 0,
    ):
        newData = newData.astype(np.float32) / self.hps.data.max_wav_value

        if self.audio_buffer is not None:
            self.audio_buffer = np.concatenate(
                [self.audio_buffer, newData], 0
            )  # 過去のデータに連結
        else:
            self.audio_buffer = newData

        convertSize = (
            inputSize + crossfadeSize + solaSearchFrame + self.settings.extraConvertSize
        )

        if convertSize % self.hps.data.hop_length != 0:  # モデルの出力のホップサイズで切り捨てが発生するので補う。
            convertSize = convertSize + (
                self.hps.data.hop_length - (convertSize % self.hps.data.hop_length)
            )

        convertOffset = -1 * convertSize
        self.audio_buffer = self.audio_buffer[convertOffset:]  # 変換対象の部分だけ抽出

        cropOffset = -1 * (inputSize + crossfadeSize)
        cropEnd = -1 * (crossfadeSize)
        crop = self.audio_buffer[cropOffset:cropEnd]

        rms = np.sqrt(np.square(crop).mean(axis=0))
        vol = max(rms, self.prevVol * 0.0)
        self.prevVol = vol

        c, f0, uv = self.get_unit_f0(self.audio_buffer, self.settings.tran)
        return (c, f0, uv, convertSize, vol)

    def _onnx_inference(self, data):
        convertSize = data[3]
        vol = data[4]
        data = (
            data[0],
            data[1],
            data[2],
        )

        if vol < self.settings.silentThreshold:
            return np.zeros(convertSize).astype(np.int16)

        c, f0, uv = [x.numpy() for x in data]
        sid_target = torch.LongTensor([self.settings.dstId]).unsqueeze(0).numpy()
        audio1 = (
            self.onnx_session.run(
                ["audio"],
                {
                    "c": c.astype(np.float32),
                    "f0": f0.astype(np.float32),
                    "uv": uv.astype(np.float32),
                    "g": sid_target.astype(np.int64),
                    "noise_scale": np.array([self.settings.noiseScale]).astype(
                        np.float32
                    ),
                    # "predict_f0": np.array([self.settings.dstId]).astype(np.int64),
                },
            )[0][0, 0]
            * self.hps.data.max_wav_value
        )

        audio1 = audio1 * vol
        result = audio1
        return result

    def _pyTorch_inference(self, data):
        if self.settings.gpu < 0 or self.gpu_num == 0:
            dev = torch.device("cpu")
        else:
            dev = torch.device("cuda", index=self.settings.gpu)

        convertSize = data[3]
        vol = data[4]
        data = (
            data[0],
            data[1],
            data[2],
        )

        if vol < self.settings.silentThreshold:
            return np.zeros(convertSize).astype(np.int16)

        with torch.no_grad():
            c, f0, uv = [x.to(dev) for x in data]
            sid_target = torch.LongTensor([self.settings.dstId]).to(dev).unsqueeze(0)
            self.net_g.to(dev)
            # audio1 = self.net_g.infer(c, f0=f0, g=sid_target, uv=uv, predict_f0=True, noice_scale=0.1)[0][0, 0].data.float()
            predict_f0_flag = True if self.settings.predictF0 == 1 else False
            audio1 = self.net_g.infer(
                c,
                f0=f0,
                g=sid_target,
                uv=uv,
                predict_f0=predict_f0_flag,
                noice_scale=self.settings.noiseScale,
            )
            audio1 = audio1[0][0].data.float()
            # audio1 = self.net_g.infer(c, f0=f0, g=sid_target, uv=uv, predict_f0=predict_f0_flag,
            #                           noice_scale=self.settings.noiceScale)[0][0, 0].data.float()
            audio1 = audio1 * self.hps.data.max_wav_value

            audio1 = audio1 * vol

            result = audio1.float().cpu().numpy()

            # result = infer_tool.pad_array(result, length)
        return result

    def inference(self, data):
        if self.slotInfo.isONNX:
            audio = self._onnx_inference(data)
        else:
            audio = self._pyTorch_inference(data)

        return audio

    def __del__(self):
        del self.net_g
        del self.onnx_session
        remove_path = os.path.join("so-vits-svc-40")
        sys.path = [x for x in sys.path if x.endswith(remove_path) is False]

        for key in list(sys.modules):
            val = sys.modules.get(key)
            try:
                file_path = val.__file__
                if file_path.find("so-vits-svc-40" + os.path.sep) >= 0:
                    # print("remove", key, file_path)
                    sys.modules.pop(key)
            except Exception:  # type:ignore
                pass

    def get_model_current(self):
        return []


def resize_f0(x, target_len):
    source = np.array(x)
    source[source < 0.001] = np.nan
    target = np.interp(
        np.arange(0, len(source) * target_len, len(source)) / target_len,
        np.arange(0, len(source)),
        source,
    )
    res = np.nan_to_num(target)
    return res


def compute_f0_dio(wav_numpy, p_len=None, sampling_rate=44100, hop_length=512):
    if p_len is None:
        p_len = wav_numpy.shape[0] // hop_length
    f0, t = pw.dio(
        wav_numpy.astype(np.double),
        fs=sampling_rate,
        f0_ceil=800,
        frame_period=1000 * hop_length / sampling_rate,
    )
    f0 = pw.stonemask(wav_numpy.astype(np.double), f0, t, sampling_rate)
    for index, pitch in enumerate(f0):
        f0[index] = round(pitch, 1)
    return resize_f0(f0, p_len)


def compute_f0_harvest(wav_numpy, p_len=None, sampling_rate=44100, hop_length=512):
    if p_len is None:
        p_len = wav_numpy.shape[0] // hop_length
    f0, t = pw.harvest(
        wav_numpy.astype(np.double),
        fs=sampling_rate,
        frame_period=5.5,
        f0_floor=71.0,
        f0_ceil=1000.0,
    )

    for index, pitch in enumerate(f0):
        f0[index] = round(pitch, 1)
    return resize_f0(f0, p_len)


def get_hubert_content_layer9(hmodel, wav_16k_tensor):
    feats = wav_16k_tensor
    if feats.dim() == 2:  # double channels
        feats = feats.mean(-1)
    assert feats.dim() == 1, feats.dim()
    feats = feats.view(1, -1)
    padding_mask = torch.BoolTensor(feats.shape).fill_(False)
    inputs = {
        "source": feats.to(wav_16k_tensor.device),
        "padding_mask": padding_mask.to(wav_16k_tensor.device),
        "output_layer": 9,  # layer 9
    }
    with torch.no_grad():
        logits = hmodel.extract_features(**inputs)

    return logits[0].transpose(1, 2)