mirror of
https://github.com/w-okada/voice-changer.git
synced 2025-02-02 16:23:58 +03:00
WIP: support so-vits-svc 40v2. can not store content vec in indexeddb.
This commit is contained in:
wataru
parent
d5cc5c9e28
commit
76ddef5ee1
6
client/demo_so-vits-svc_40v2/dist/index.js
vendored
6
client/demo_so-vits-svc_40v2/dist/index.js
vendored
File diff suppressed because one or more lines are too long
@ -86,6 +86,26 @@ export const useModelSettingArea = (): ServerSettingState => {
|
||||
})
|
||||
}
|
||||
|
||||
const onClusterFileLoadClicked = async () => {
|
||||
const file = await fileSelector("")
|
||||
if (file.name.endsWith(".pth") == false) {
|
||||
alert("モデルファイルの拡張子はpthである必要があります。")
|
||||
return
|
||||
}
|
||||
appState.serverSetting.setFileUploadSetting({
|
||||
...appState.serverSetting.fileUploadSetting,
|
||||
clusterTorchModel: {
|
||||
file: file
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
const onClusterFileClearClicked = () => {
|
||||
appState.serverSetting.setFileUploadSetting({
|
||||
...appState.serverSetting.fileUploadSetting,
|
||||
clusterTorchModel: null
|
||||
})
|
||||
}
|
||||
// const onOnnxFileLoadClicked = async () => {
|
||||
// const file = await fileSelector("")
|
||||
// if (file.name.endsWith(".onnx") == false) {
|
||||
@ -116,6 +136,7 @@ export const useModelSettingArea = (): ServerSettingState => {
|
||||
|
||||
const configFilenameText = appState.serverSetting.fileUploadSetting.configFile?.filename || appState.serverSetting.fileUploadSetting.configFile?.file?.name || ""
|
||||
const hubertModelFilenameText = appState.serverSetting.fileUploadSetting.hubertTorchModel?.filename || appState.serverSetting.fileUploadSetting.hubertTorchModel?.file?.name || ""
|
||||
const clusterModelFilenameText = appState.serverSetting.fileUploadSetting.clusterTorchModel?.filename || appState.serverSetting.fileUploadSetting.clusterTorchModel?.file?.name || ""
|
||||
// const onnxModelFilenameText = appState.serverSetting.fileUploadSetting.onnxModel?.filename || appState.serverSetting.fileUploadSetting.onnxModel?.file?.name || ""
|
||||
|
||||
const pyTorchFilenameText = appState.serverSetting.fileUploadSetting.pyTorchModel?.filename || appState.serverSetting.fileUploadSetting.pyTorchModel?.file?.name || ""
|
||||
@ -159,6 +180,16 @@ export const useModelSettingArea = (): ServerSettingState => {
|
||||
<div className="body-button left-margin-1" onClick={onHubertFileClearClicked}>clear</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className="body-row split-3-3-4 left-padding-1 guided">
|
||||
<div className="body-item-title left-padding-2">cluster(.pth)</div>
|
||||
<div className="body-item-text">
|
||||
<div>{clusterModelFilenameText}</div>
|
||||
</div>
|
||||
<div className="body-button-container">
|
||||
<div className="body-button" onClick={onClusterFileLoadClicked}>select</div>
|
||||
<div className="body-button left-margin-1" onClick={onClusterFileClearClicked}>clear</div>
|
||||
</div>
|
||||
</div>
|
||||
{/* <div className="body-row split-3-3-4 left-padding-1 guided">
|
||||
<div className="body-item-title left-padding-2">Onnx(.onnx)</div>
|
||||
<div className="body-item-text">
|
||||
|
||||
@ -66,6 +66,26 @@ export const useSpeakerSetting = () => {
|
||||
])
|
||||
|
||||
|
||||
const clusterInferRatioRow = useMemo(() => {
|
||||
return (
|
||||
<div className="body-row split-3-3-4 left-padding-1 guided">
|
||||
<div className="body-item-title left-padding-1 ">Cluster infer ratio</div>
|
||||
<div>
|
||||
<input type="range" className="body-item-input-slider" min="0" max="1" step="0.1" value={appState.serverSetting.serverSetting.clusterInferRatio} onChange={(e) => {
|
||||
appState.serverSetting.updateServerSettings({ ...appState.serverSetting.serverSetting, clusterInferRatio: Number(e.target.value) })
|
||||
}}></input>
|
||||
<span className="body-item-input-slider-val">{appState.serverSetting.serverSetting.clusterInferRatio}</span>
|
||||
</div>
|
||||
<div className="body-button-container">
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}, [
|
||||
appState.serverSetting.serverSetting,
|
||||
appState.serverSetting.updateServerSettings
|
||||
])
|
||||
|
||||
|
||||
const noiceScaleRow = useMemo(() => {
|
||||
return (
|
||||
<div className="body-row split-3-3-4 left-padding-1 guided">
|
||||
@ -122,6 +142,7 @@ export const useSpeakerSetting = () => {
|
||||
<div className="partition-content">
|
||||
{dstIdRow}
|
||||
{tranRow}
|
||||
{clusterInferRatioRow}
|
||||
{noiceScaleRow}
|
||||
{silentThresholdRow}
|
||||
</div>
|
||||
|
||||
@ -109,7 +109,7 @@ export class ServerConfigurator {
|
||||
})
|
||||
}
|
||||
|
||||
loadModel = async (configFilename: string, pyTorchModelFilename: string | null, onnxModelFilename: string | null, hubertTorchModelFilename: string | null) => {
|
||||
loadModel = async (configFilename: string, pyTorchModelFilename: string | null, onnxModelFilename: string | null, hubertTorchModelFilename: string | null, clusterTorchModelFilename: string | null) => {
|
||||
const url = this.serverUrl + "/load_model"
|
||||
const info = new Promise<ServerInfo>(async (resolve) => {
|
||||
const formData = new FormData();
|
||||
@ -117,6 +117,8 @@ export class ServerConfigurator {
|
||||
formData.append("onnxModelFilename", onnxModelFilename || "-");
|
||||
formData.append("configFilename", configFilename);
|
||||
formData.append("hubertTorchModelFilename", hubertTorchModelFilename || "-");
|
||||
formData.append("clusterTorchModelFilename", clusterTorchModelFilename || "-");
|
||||
|
||||
|
||||
const request = new Request(url, {
|
||||
method: 'POST',
|
||||
|
||||
@ -243,8 +243,8 @@ export class VoiceChangerClient {
|
||||
concatUploadedFile = (filename: string, chunkNum: number) => {
|
||||
return this.configurator.concatUploadedFile(filename, chunkNum)
|
||||
}
|
||||
loadModel = (configFilename: string, pyTorchModelFilename: string | null, onnxModelFilename: string | null, hubertTorchModelFilename: string | null) => {
|
||||
return this.configurator.loadModel(configFilename, pyTorchModelFilename, onnxModelFilename, hubertTorchModelFilename)
|
||||
loadModel = (configFilename: string, pyTorchModelFilename: string | null, onnxModelFilename: string | null, hubertTorchModelFilename: string | null, clusterTorchModelFilename: string | null) => {
|
||||
return this.configurator.loadModel(configFilename, pyTorchModelFilename, onnxModelFilename, hubertTorchModelFilename, clusterTorchModelFilename)
|
||||
}
|
||||
|
||||
//## Worklet ##//
|
||||
|
||||
@ -71,6 +71,7 @@ export const ServerSettingKey = {
|
||||
"predictF0": "predictF0",
|
||||
"silentThreshold": "silentThreshold",
|
||||
"extraConvertSize": "extraConvertSize",
|
||||
"clusterInferRatio": "clusterInferRatio",
|
||||
|
||||
"inputSampleRate": "inputSampleRate",
|
||||
} as const
|
||||
@ -98,6 +99,7 @@ export type VoiceChangerServerSetting = {
|
||||
predictF0: number // so-vits-svc
|
||||
silentThreshold: number // so-vits-svc
|
||||
extraConvertSize: number// so-vits-svc
|
||||
clusterInferRatio: number // so-vits-svc
|
||||
|
||||
inputSampleRate: InputSampleRate
|
||||
}
|
||||
@ -130,6 +132,7 @@ export const DefaultServerSetting_MMVCv15: ServerInfo = {
|
||||
predictF0: 0,
|
||||
silentThreshold: 0,
|
||||
extraConvertSize: 0,
|
||||
clusterInferRatio: 0,
|
||||
|
||||
inputSampleRate: 24000,
|
||||
|
||||
@ -161,6 +164,7 @@ export const DefaultServerSetting_MMVCv13: ServerInfo = {
|
||||
predictF0: 0,
|
||||
silentThreshold: 0,
|
||||
extraConvertSize: 0,
|
||||
clusterInferRatio: 0,
|
||||
|
||||
inputSampleRate: 24000,
|
||||
|
||||
@ -196,6 +200,7 @@ export const DefaultServerSetting_so_vits_svc_40v2: ServerInfo = {
|
||||
predictF0: 0,
|
||||
silentThreshold: 0.00001,
|
||||
extraConvertSize: 1024 * 32,
|
||||
clusterInferRatio: 0.1,
|
||||
|
||||
inputSampleRate: 24000,
|
||||
|
||||
|
||||
@ -15,13 +15,15 @@ export type FileUploadSetting = {
|
||||
onnxModel: ModelData | null
|
||||
configFile: ModelData | null
|
||||
hubertTorchModel: ModelData | null
|
||||
clusterTorchModel: ModelData | null
|
||||
}
|
||||
|
||||
const InitialFileUploadSetting: FileUploadSetting = {
|
||||
pyTorchModel: null,
|
||||
configFile: null,
|
||||
onnxModel: null,
|
||||
hubertTorchModel: null
|
||||
hubertTorchModel: null,
|
||||
clusterTorchModel: null
|
||||
}
|
||||
|
||||
export type UseServerSettingProps = {
|
||||
@ -172,9 +174,12 @@ export const useServerSetting = (props: UseServerSettingProps): ServerSettingSta
|
||||
fileUploadSetting.hubertTorchModel!.data = await fileUploadSetting.hubertTorchModel!.file!.arrayBuffer()
|
||||
fileUploadSetting.hubertTorchModel!.filename = await fileUploadSetting.hubertTorchModel!.file!.name
|
||||
}
|
||||
|
||||
if (props.clientType == "so_vits_svc_40v2c" && !fileUploadSetting.clusterTorchModel!.data) {
|
||||
fileUploadSetting.clusterTorchModel!.data = await fileUploadSetting.clusterTorchModel!.file!.arrayBuffer()
|
||||
fileUploadSetting.clusterTorchModel!.filename = await fileUploadSetting.clusterTorchModel!.file!.name
|
||||
}
|
||||
// ファイルをサーバにアップロード
|
||||
const models = [fileUploadSetting.onnxModel, fileUploadSetting.pyTorchModel, fileUploadSetting.hubertTorchModel].filter(x => { return x != null }) as ModelData[]
|
||||
const models = [fileUploadSetting.onnxModel, fileUploadSetting.pyTorchModel, fileUploadSetting.hubertTorchModel, fileUploadSetting.clusterTorchModel].filter(x => { return x != null }) as ModelData[]
|
||||
for (let i = 0; i < models.length; i++) {
|
||||
const progRate = 1 / models.length
|
||||
const progOffset = 100 * i * progRate
|
||||
@ -188,20 +193,27 @@ export const useServerSetting = (props: UseServerSettingProps): ServerSettingSta
|
||||
console.log(progress, end)
|
||||
})
|
||||
|
||||
const loadPromise = props.voiceChangerClient.loadModel(fileUploadSetting.configFile.filename!, fileUploadSetting.pyTorchModel?.filename || null, fileUploadSetting.onnxModel?.filename || null, fileUploadSetting.hubertTorchModel?.filename || null)
|
||||
const loadPromise = props.voiceChangerClient.loadModel(fileUploadSetting.configFile.filename!, fileUploadSetting.pyTorchModel?.filename || null, fileUploadSetting.onnxModel?.filename || null, fileUploadSetting.hubertTorchModel?.filename || null, fileUploadSetting.clusterTorchModel?.filename || null)
|
||||
|
||||
// サーバでロード中にキャッシュにセーブ
|
||||
try {
|
||||
const saveData: FileUploadSetting = {
|
||||
pyTorchModel: fileUploadSetting.pyTorchModel ? { data: fileUploadSetting.pyTorchModel.data, filename: fileUploadSetting.pyTorchModel.filename } : null,
|
||||
onnxModel: fileUploadSetting.onnxModel ? { data: fileUploadSetting.onnxModel.data, filename: fileUploadSetting.onnxModel.filename } : null,
|
||||
configFile: { data: fileUploadSetting.configFile.data, filename: fileUploadSetting.configFile.filename },
|
||||
hubertTorchModel: fileUploadSetting.hubertTorchModel ? {
|
||||
data: fileUploadSetting.hubertTorchModel.data, filename: fileUploadSetting.hubertTorchModel.filename
|
||||
} : null,
|
||||
clusterTorchModel: fileUploadSetting.clusterTorchModel ? {
|
||||
data: fileUploadSetting.clusterTorchModel.data, filename: fileUploadSetting.clusterTorchModel.filename
|
||||
} : null
|
||||
|
||||
}
|
||||
setItem(INDEXEDDB_KEY_MODEL_DATA, saveData)
|
||||
|
||||
} catch (e) {
|
||||
console.log("Excpetion:::::::::", e)
|
||||
}
|
||||
|
||||
await loadPromise
|
||||
setUploadProgress(0)
|
||||
setIsUploading(false)
|
||||
|
||||
@ -40,6 +40,7 @@ def setupArgParser():
|
||||
parser.add_argument("--modelType", type=str,
|
||||
default="MMVCv15", help="model type: MMVCv13, MMVCv15, so-vits-svc-40v2")
|
||||
parser.add_argument("--hubert", type=str, help="path to hubert model")
|
||||
parser.add_argument("--cluster", type=str, help="path to cluster model")
|
||||
|
||||
return parser
|
||||
|
||||
@ -81,6 +82,7 @@ CONFIG = args.c
|
||||
MODEL = args.m if args.m != None else None
|
||||
ONNX_MODEL = args.o if args.o != None else None
|
||||
HUBERT_MODEL = args.hubert if args.hubert != None else None
|
||||
CLUSTER_MODEL = args.cluster if args.cluster != None else None
|
||||
MODEL_TYPE = os.environ.get('MODEL_TYPE', None)
|
||||
if MODEL_TYPE == None:
|
||||
MODEL_TYPE = args.modelType
|
||||
@ -103,9 +105,9 @@ if args.colab == True:
|
||||
voiceChangerManager = VoiceChangerManager.get_instance()
|
||||
if CONFIG and (MODEL or ONNX_MODEL):
|
||||
if MODEL_TYPE == "MMVCv15" or MODEL_TYPE == "MMVCv13":
|
||||
voiceChangerManager.loadModel(CONFIG, MODEL, ONNX_MODEL, None)
|
||||
voiceChangerManager.loadModel(CONFIG, MODEL, ONNX_MODEL, None, None)
|
||||
else:
|
||||
voiceChangerManager.loadModel(CONFIG, MODEL, ONNX_MODEL, HUBERT_MODEL)
|
||||
voiceChangerManager.loadModel(CONFIG, MODEL, ONNX_MODEL, HUBERT_MODEL, CLUSTER_MODEL)
|
||||
|
||||
app_fastapi = MMVC_Rest.get_instance(voiceChangerManager)
|
||||
app_socketio = MMVC_SocketIOApp.get_instance(app_fastapi, voiceChangerManager)
|
||||
|
||||
@ -54,15 +54,18 @@ class MMVC_Rest_Fileuploader:
|
||||
pyTorchModelFilename: str = Form(...),
|
||||
onnxModelFilename: str = Form(...),
|
||||
configFilename: str = Form(...),
|
||||
hubertTorchModelFilename: str = Form(...)
|
||||
hubertTorchModelFilename: str = Form(...),
|
||||
clusterTorchModelFilename: str = Form(...)
|
||||
):
|
||||
print("Hubert:", hubertTorchModelFilename)
|
||||
pyTorchModelFilePath = os.path.join(UPLOAD_DIR, pyTorchModelFilename) if pyTorchModelFilename != "-" else None
|
||||
onnxModelFilePath = os.path.join(UPLOAD_DIR, onnxModelFilename) if onnxModelFilename != "-" else None
|
||||
configFilePath = os.path.join(UPLOAD_DIR, configFilename)
|
||||
hubertTorchModelFilePath = os.path.join(UPLOAD_DIR, hubertTorchModelFilename) if hubertTorchModelFilename != "-" else None
|
||||
clusterTorchModelFilePath = os.path.join(UPLOAD_DIR, clusterTorchModelFilename) if clusterTorchModelFilename != "-" else None
|
||||
|
||||
info = self.voiceChangerManager.loadModel(configFilePath, pyTorchModelFilePath, onnxModelFilePath, hubertTorchModelFilePath)
|
||||
info = self.voiceChangerManager.loadModel(configFilePath, pyTorchModelFilePath, onnxModelFilePath,
|
||||
hubertTorchModelFilePath, clusterTorchModelFilePath)
|
||||
json_compatible_item_data = jsonable_encoder(info)
|
||||
return JSONResponse(content=json_compatible_item_data)
|
||||
# return {"load": f"{configFilePath}, {pyTorchModelFilePath}, {onnxModelFilePath}"}
|
||||
|
||||
@ -19,6 +19,7 @@ import onnxruntime
|
||||
import pyworld as pw
|
||||
|
||||
from models import SynthesizerTrn
|
||||
import cluster
|
||||
import utils
|
||||
from fairseq import checkpoint_utils
|
||||
import librosa
|
||||
@ -36,6 +37,7 @@ class SoVitsSvc40v2Settings():
|
||||
predictF0: int = 0 # 0:False, 1:True
|
||||
silentThreshold: float = 0.00001
|
||||
extraConvertSize: int = 1024 * 32
|
||||
clusterInferRatio: float = 0.1
|
||||
|
||||
framework: str = "PyTorch" # PyTorch or ONNX
|
||||
pyTorchModelFile: str = ""
|
||||
@ -44,7 +46,7 @@ class SoVitsSvc40v2Settings():
|
||||
|
||||
# ↓mutableな物だけ列挙
|
||||
intData = ["gpu", "dstId", "tran", "predictF0", "extraConvertSize"]
|
||||
floatData = ["noiceScale", "silentThreshold"]
|
||||
floatData = ["noiceScale", "silentThreshold", "clusterInferRatio"]
|
||||
strData = ["framework", "f0Detector"]
|
||||
|
||||
|
||||
@ -58,20 +60,40 @@ class SoVitsSvc40v2:
|
||||
self.gpu_num = torch.cuda.device_count()
|
||||
self.prevVol = 0
|
||||
|
||||
def loadModel(self, config: str, pyTorch_model_file: str = None, onnx_model_file: str = None, hubertTorchModel: str = None):
|
||||
def loadModel(self, config: str, pyTorch_model_file: str = None, onnx_model_file: str = None, hubertTorchModel: str = None, clusterTorchModel: str = None):
|
||||
self.settings.configFile = config
|
||||
self.hps = utils.get_hparams_from_file(config)
|
||||
|
||||
# hubert model
|
||||
# vec_path = "hubert/checkpoint_best_legacy_500.pt"
|
||||
try:
|
||||
vec_path = hubertTorchModel
|
||||
print("hubert 1 ", hubertTorchModel)
|
||||
models, saved_cfg, task = checkpoint_utils.load_model_ensemble_and_task(
|
||||
[vec_path],
|
||||
suffix="",
|
||||
)
|
||||
print("hubert 2 ", hubertTorchModel)
|
||||
model = models[0]
|
||||
print("hubert 3 ", hubertTorchModel)
|
||||
model.eval()
|
||||
self.hubert_model = utils.get_hubert_model().cpu()
|
||||
print("hubert 4 ", hubertTorchModel)
|
||||
self.hubert_model = model.cpu()
|
||||
print("hubert 5 ", hubertTorchModel)
|
||||
except Exception as e:
|
||||
print("EXCEPTION1", e)
|
||||
|
||||
# cluster
|
||||
try:
|
||||
if os.path.exists(clusterTorchModel):
|
||||
print("load kmean11", clusterTorchModel)
|
||||
self.cluster_model = cluster.get_cluster_model(clusterTorchModel)
|
||||
print("load kmean12", clusterTorchModel)
|
||||
else:
|
||||
print("load kmean21", clusterTorchModel)
|
||||
self.cluster_model = None
|
||||
print("load kmean22", clusterTorchModel)
|
||||
except Exception as e:
|
||||
print("EXCEPTION2", e)
|
||||
|
||||
if pyTorch_model_file != None:
|
||||
self.settings.pyTorchModelFile = pyTorch_model_file
|
||||
@ -157,6 +179,14 @@ class SoVitsSvc40v2:
|
||||
wav16k = torch.from_numpy(wav16k)
|
||||
c = utils.get_hubert_content(self.hubert_model, wav_16k_tensor=wav16k)
|
||||
c = utils.repeat_expand_2d(c.squeeze(0), f0.shape[1])
|
||||
|
||||
if self.settings.clusterInferRatio != 0 and self.cluster_model != None:
|
||||
# self.hsp.spk.tsukuyomi
|
||||
cluster_c = cluster.get_cluster_center_result(self.cluster_model, c.cpu().numpy().T, "tsukuyomi").T
|
||||
# cluster_c = cluster.get_cluster_center_result(self.cluster_model, c.cpu().numpy().T, self.settings.dstId).T
|
||||
cluster_c = torch.FloatTensor(cluster_c).cpu()
|
||||
c = self.settings.clusterInferRatio * cluster_c + (1 - self.settings.clusterInferRatio) * c
|
||||
|
||||
c = c.unsqueeze(0)
|
||||
return c, f0, uv
|
||||
|
||||
|
||||
@ -70,11 +70,11 @@ class VoiceChanger():
|
||||
|
||||
print(f"VoiceChanger Initialized (GPU_NUM:{self.gpu_num}, mps_enabled:{self.mps_enabled})")
|
||||
|
||||
def loadModel(self, config: str, pyTorch_model_file: str = None, onnx_model_file: str = None, hubertTorchModel: str = None):
|
||||
def loadModel(self, config: str, pyTorch_model_file: str = None, onnx_model_file: str = None, hubertTorchModel: str = None, clusterTorchModel: str = None):
|
||||
if self.modelType == "MMVCv15" or self.modelType == "MMVCv13":
|
||||
return self.voiceChanger.loadModel(config, pyTorch_model_file, onnx_model_file)
|
||||
else: # so-vits-svc-40v2
|
||||
return self.voiceChanger.loadModel(config, pyTorch_model_file, onnx_model_file, hubertTorchModel)
|
||||
return self.voiceChanger.loadModel(config, pyTorch_model_file, onnx_model_file, hubertTorchModel, clusterTorchModel)
|
||||
|
||||
def get_info(self):
|
||||
data = asdict(self.settings)
|
||||
|
||||
@ -10,8 +10,8 @@ class VoiceChangerManager():
|
||||
cls._instance.voiceChanger = VoiceChanger()
|
||||
return cls._instance
|
||||
|
||||
def loadModel(self, config, model, onnx_model, hubertTorchModel):
|
||||
info = self.voiceChanger.loadModel(config, model, onnx_model, hubertTorchModel)
|
||||
def loadModel(self, config, model, onnx_model, hubertTorchModel, clusterTorchModel):
|
||||
info = self.voiceChanger.loadModel(config, model, onnx_model, hubertTorchModel, clusterTorchModel)
|
||||
info["status"] = "OK"
|
||||
return info
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user