WIP: support so-vits-svc 40v2. can not store content vec in indexeddb.

client/demo_so-vits-svc_40v2/src/102_model_setting.tsx

@@ -86,6 +86,26 @@ export const useModelSettingArea = (): ServerSettingState => {
             })
         }
 
+        const onClusterFileLoadClicked = async () => {
+            const file = await fileSelector("")
+            if (file.name.endsWith(".pth") == false) {
+                alert("モデルファイルの拡張子はpthである必要があります。")
+                return
+            }
+            appState.serverSetting.setFileUploadSetting({
+                ...appState.serverSetting.fileUploadSetting,
+                clusterTorchModel: {
+                    file: file
+                }
+            })
+        }
+
+        const onClusterFileClearClicked = () => {
+            appState.serverSetting.setFileUploadSetting({
+                ...appState.serverSetting.fileUploadSetting,
+                clusterTorchModel: null
+            })
+        }
         // const onOnnxFileLoadClicked = async () => {
         //     const file = await fileSelector("")
         //     if (file.name.endsWith(".onnx") == false) {
@@ -116,6 +136,7 @@ export const useModelSettingArea = (): ServerSettingState => {
 
         const configFilenameText = appState.serverSetting.fileUploadSetting.configFile?.filename || appState.serverSetting.fileUploadSetting.configFile?.file?.name || ""
         const hubertModelFilenameText = appState.serverSetting.fileUploadSetting.hubertTorchModel?.filename || appState.serverSetting.fileUploadSetting.hubertTorchModel?.file?.name || ""
+        const clusterModelFilenameText = appState.serverSetting.fileUploadSetting.clusterTorchModel?.filename || appState.serverSetting.fileUploadSetting.clusterTorchModel?.file?.name || ""
         // const onnxModelFilenameText = appState.serverSetting.fileUploadSetting.onnxModel?.filename || appState.serverSetting.fileUploadSetting.onnxModel?.file?.name || ""
 
         const pyTorchFilenameText = appState.serverSetting.fileUploadSetting.pyTorchModel?.filename || appState.serverSetting.fileUploadSetting.pyTorchModel?.file?.name || ""
@@ -159,6 +180,16 @@ export const useModelSettingArea = (): ServerSettingState => {
                         <div className="body-button left-margin-1" onClick={onHubertFileClearClicked}>clear</div>
                     </div>
                 </div>
+                <div className="body-row split-3-3-4 left-padding-1 guided">
+                    <div className="body-item-title left-padding-2">cluster(.pth)</div>
+                    <div className="body-item-text">
+                        <div>{clusterModelFilenameText}</div>
+                    </div>
+                    <div className="body-button-container">
+                        <div className="body-button" onClick={onClusterFileLoadClicked}>select</div>
+                        <div className="body-button left-margin-1" onClick={onClusterFileClearClicked}>clear</div>
+                    </div>
+                </div>
                 {/* <div className="body-row split-3-3-4 left-padding-1 guided">
                     <div className="body-item-title left-padding-2">Onnx(.onnx)</div>
                     <div className="body-item-text">

client/demo_so-vits-svc_40v2/src/105_speaker_setting.tsx

@@ -66,6 +66,26 @@ export const useSpeakerSetting = () => {
     ])
 
 
+    const clusterInferRatioRow = useMemo(() => {
+        return (
+            <div className="body-row split-3-3-4 left-padding-1 guided">
+                <div className="body-item-title left-padding-1 ">Cluster infer ratio</div>
+                <div>
+                    <input type="range" className="body-item-input-slider" min="0" max="1" step="0.1" value={appState.serverSetting.serverSetting.clusterInferRatio} onChange={(e) => {
+                        appState.serverSetting.updateServerSettings({ ...appState.serverSetting.serverSetting, clusterInferRatio: Number(e.target.value) })
+                    }}></input>
+                    <span className="body-item-input-slider-val">{appState.serverSetting.serverSetting.clusterInferRatio}</span>
+                </div>
+                <div className="body-button-container">
+                </div>
+            </div>
+        )
+    }, [
+        appState.serverSetting.serverSetting,
+        appState.serverSetting.updateServerSettings
+    ])
+
+
     const noiceScaleRow = useMemo(() => {
         return (
             <div className="body-row split-3-3-4 left-padding-1 guided">
@@ -122,6 +142,7 @@ export const useSpeakerSetting = () => {
                     <div className="partition-content">
                         {dstIdRow}
                         {tranRow}
+                        {clusterInferRatioRow}
                         {noiceScaleRow}
                         {silentThresholdRow}
                     </div>

client/lib/src/ServerConfigurator.ts

@@ -109,7 +109,7 @@ export class ServerConfigurator {
         })
     }
 
-    loadModel = async (configFilename: string, pyTorchModelFilename: string | null, onnxModelFilename: string | null, hubertTorchModelFilename: string | null) => {
+    loadModel = async (configFilename: string, pyTorchModelFilename: string | null, onnxModelFilename: string | null, hubertTorchModelFilename: string | null, clusterTorchModelFilename: string | null) => {
         const url = this.serverUrl + "/load_model"
         const info = new Promise<ServerInfo>(async (resolve) => {
             const formData = new FormData();
@@ -117,6 +117,8 @@ export class ServerConfigurator {
             formData.append("onnxModelFilename", onnxModelFilename || "-");
             formData.append("configFilename", configFilename);
             formData.append("hubertTorchModelFilename", hubertTorchModelFilename || "-");
+            formData.append("clusterTorchModelFilename", clusterTorchModelFilename || "-");
+
 
             const request = new Request(url, {
                 method: 'POST',

client/lib/src/VoiceChangerClient.ts

@@ -243,8 +243,8 @@ export class VoiceChangerClient {
     concatUploadedFile = (filename: string, chunkNum: number) => {
         return this.configurator.concatUploadedFile(filename, chunkNum)
     }
-    loadModel = (configFilename: string, pyTorchModelFilename: string | null, onnxModelFilename: string | null, hubertTorchModelFilename: string | null) => {
+    loadModel = (configFilename: string, pyTorchModelFilename: string | null, onnxModelFilename: string | null, hubertTorchModelFilename: string | null, clusterTorchModelFilename: string | null) => {
-        return this.configurator.loadModel(configFilename, pyTorchModelFilename, onnxModelFilename, hubertTorchModelFilename)
+        return this.configurator.loadModel(configFilename, pyTorchModelFilename, onnxModelFilename, hubertTorchModelFilename, clusterTorchModelFilename)
     }
 
     //##  Worklet ##//

client/lib/src/const.ts

@@ -71,6 +71,7 @@ export const ServerSettingKey = {
     "predictF0": "predictF0",
     "silentThreshold": "silentThreshold",
     "extraConvertSize": "extraConvertSize",
+    "clusterInferRatio": "clusterInferRatio",
 
     "inputSampleRate": "inputSampleRate",
 } as const
@@ -98,6 +99,7 @@ export type VoiceChangerServerSetting = {
     predictF0: number // so-vits-svc
     silentThreshold: number // so-vits-svc
     extraConvertSize: number// so-vits-svc
+    clusterInferRatio: number // so-vits-svc
 
     inputSampleRate: InputSampleRate
 }
@@ -130,6 +132,7 @@ export const DefaultServerSetting_MMVCv15: ServerInfo = {
     predictF0: 0,
     silentThreshold: 0,
     extraConvertSize: 0,
+    clusterInferRatio: 0,
 
     inputSampleRate: 24000,
 
@@ -161,6 +164,7 @@ export const DefaultServerSetting_MMVCv13: ServerInfo = {
     predictF0: 0,
     silentThreshold: 0,
     extraConvertSize: 0,
+    clusterInferRatio: 0,
 
     inputSampleRate: 24000,
 
@@ -196,6 +200,7 @@ export const DefaultServerSetting_so_vits_svc_40v2: ServerInfo = {
     predictF0: 0,
     silentThreshold: 0.00001,
     extraConvertSize: 1024 * 32,
+    clusterInferRatio: 0.1,
 
     inputSampleRate: 24000,
 

client/lib/src/hooks/useServerSetting.ts

@@ -15,13 +15,15 @@ export type FileUploadSetting = {
     onnxModel: ModelData | null
     configFile: ModelData | null
     hubertTorchModel: ModelData | null
+    clusterTorchModel: ModelData | null
 }
 
 const InitialFileUploadSetting: FileUploadSetting = {
     pyTorchModel: null,
     configFile: null,
     onnxModel: null,
-    hubertTorchModel: null
+    hubertTorchModel: null,
+    clusterTorchModel: null
 }
 
 export type UseServerSettingProps = {
@@ -172,9 +174,12 @@ export const useServerSetting = (props: UseServerSettingProps): ServerSettingSta
                 fileUploadSetting.hubertTorchModel!.data = await fileUploadSetting.hubertTorchModel!.file!.arrayBuffer()
                 fileUploadSetting.hubertTorchModel!.filename = await fileUploadSetting.hubertTorchModel!.file!.name
             }
-
+            if (props.clientType == "so_vits_svc_40v2c" && !fileUploadSetting.clusterTorchModel!.data) {
+                fileUploadSetting.clusterTorchModel!.data = await fileUploadSetting.clusterTorchModel!.file!.arrayBuffer()
+                fileUploadSetting.clusterTorchModel!.filename = await fileUploadSetting.clusterTorchModel!.file!.name
+            }
             // ファイルをサーバにアップロード
-            const models = [fileUploadSetting.onnxModel, fileUploadSetting.pyTorchModel, fileUploadSetting.hubertTorchModel].filter(x => { return x != null }) as ModelData[]
+            const models = [fileUploadSetting.onnxModel, fileUploadSetting.pyTorchModel, fileUploadSetting.hubertTorchModel, fileUploadSetting.clusterTorchModel].filter(x => { return x != null }) as ModelData[]
             for (let i = 0; i < models.length; i++) {
                 const progRate = 1 / models.length
                 const progOffset = 100 * i * progRate
@@ -188,19 +193,26 @@ export const useServerSetting = (props: UseServerSettingProps): ServerSettingSta
                 console.log(progress, end)
             })
 
-            const loadPromise = props.voiceChangerClient.loadModel(fileUploadSetting.configFile.filename!, fileUploadSetting.pyTorchModel?.filename || null, fileUploadSetting.onnxModel?.filename || null, fileUploadSetting.hubertTorchModel?.filename || null)
+            const loadPromise = props.voiceChangerClient.loadModel(fileUploadSetting.configFile.filename!, fileUploadSetting.pyTorchModel?.filename || null, fileUploadSetting.onnxModel?.filename || null, fileUploadSetting.hubertTorchModel?.filename || null, fileUploadSetting.clusterTorchModel?.filename || null)
 
             // サーバでロード中にキャッシュにセーブ
-            const saveData: FileUploadSetting = {
+            try {
-                pyTorchModel: fileUploadSetting.pyTorchModel ? { data: fileUploadSetting.pyTorchModel.data, filename: fileUploadSetting.pyTorchModel.filename } : null,
+                const saveData: FileUploadSetting = {
-                onnxModel: fileUploadSetting.onnxModel ? { data: fileUploadSetting.onnxModel.data, filename: fileUploadSetting.onnxModel.filename } : null,
+                    pyTorchModel: fileUploadSetting.pyTorchModel ? { data: fileUploadSetting.pyTorchModel.data, filename: fileUploadSetting.pyTorchModel.filename } : null,
-                configFile: { data: fileUploadSetting.configFile.data, filename: fileUploadSetting.configFile.filename },
+                    onnxModel: fileUploadSetting.onnxModel ? { data: fileUploadSetting.onnxModel.data, filename: fileUploadSetting.onnxModel.filename } : null,
-                hubertTorchModel: fileUploadSetting.hubertTorchModel ? {
+                    configFile: { data: fileUploadSetting.configFile.data, filename: fileUploadSetting.configFile.filename },
-                    data: fileUploadSetting.hubertTorchModel.data, filename: fileUploadSetting.hubertTorchModel.filename
+                    hubertTorchModel: fileUploadSetting.hubertTorchModel ? {
-                } : null
+                        data: fileUploadSetting.hubertTorchModel.data, filename: fileUploadSetting.hubertTorchModel.filename
+                    } : null,
+                    clusterTorchModel: fileUploadSetting.clusterTorchModel ? {
+                        data: fileUploadSetting.clusterTorchModel.data, filename: fileUploadSetting.clusterTorchModel.filename
+                    } : null
+                }
+                setItem(INDEXEDDB_KEY_MODEL_DATA, saveData)
 
+            } catch (e) {
+                console.log("Excpetion:::::::::", e)
             }
-            setItem(INDEXEDDB_KEY_MODEL_DATA, saveData)
 
             await loadPromise
             setUploadProgress(0)

server/MMVCServerSIO.py

@@ -40,6 +40,7 @@ def setupArgParser():
     parser.add_argument("--modelType", type=str,
                         default="MMVCv15", help="model type: MMVCv13, MMVCv15, so-vits-svc-40v2")
     parser.add_argument("--hubert", type=str, help="path to hubert model")
+    parser.add_argument("--cluster", type=str, help="path to cluster model")
 
     return parser
 
@@ -81,6 +82,7 @@ CONFIG = args.c
 MODEL = args.m if args.m != None else None
 ONNX_MODEL = args.o if args.o != None else None
 HUBERT_MODEL = args.hubert if args.hubert != None else None
+CLUSTER_MODEL = args.cluster if args.cluster != None else None
 MODEL_TYPE = os.environ.get('MODEL_TYPE', None)
 if MODEL_TYPE == None:
     MODEL_TYPE = args.modelType
@@ -103,9 +105,9 @@ if args.colab == True:
 voiceChangerManager = VoiceChangerManager.get_instance()
 if CONFIG and (MODEL or ONNX_MODEL):
     if MODEL_TYPE == "MMVCv15" or MODEL_TYPE == "MMVCv13":
-        voiceChangerManager.loadModel(CONFIG, MODEL, ONNX_MODEL, None)
+        voiceChangerManager.loadModel(CONFIG, MODEL, ONNX_MODEL, None, None)
     else:
-        voiceChangerManager.loadModel(CONFIG, MODEL, ONNX_MODEL, HUBERT_MODEL)
+        voiceChangerManager.loadModel(CONFIG, MODEL, ONNX_MODEL, HUBERT_MODEL, CLUSTER_MODEL)
 
 app_fastapi = MMVC_Rest.get_instance(voiceChangerManager)
 app_socketio = MMVC_SocketIOApp.get_instance(app_fastapi, voiceChangerManager)

server/restapi/MMVC_Rest_Fileuploader.py

@@ -54,15 +54,18 @@ class MMVC_Rest_Fileuploader:
         pyTorchModelFilename: str = Form(...),
         onnxModelFilename: str = Form(...),
         configFilename: str = Form(...),
-        hubertTorchModelFilename: str = Form(...)
+        hubertTorchModelFilename: str = Form(...),
+        clusterTorchModelFilename: str = Form(...)
     ):
         print("Hubert:", hubertTorchModelFilename)
         pyTorchModelFilePath = os.path.join(UPLOAD_DIR, pyTorchModelFilename) if pyTorchModelFilename != "-" else None
         onnxModelFilePath = os.path.join(UPLOAD_DIR, onnxModelFilename) if onnxModelFilename != "-" else None
         configFilePath = os.path.join(UPLOAD_DIR, configFilename)
         hubertTorchModelFilePath = os.path.join(UPLOAD_DIR, hubertTorchModelFilename) if hubertTorchModelFilename != "-" else None
+        clusterTorchModelFilePath = os.path.join(UPLOAD_DIR, clusterTorchModelFilename) if clusterTorchModelFilename != "-" else None
 
-        info = self.voiceChangerManager.loadModel(configFilePath, pyTorchModelFilePath, onnxModelFilePath, hubertTorchModelFilePath)
+        info = self.voiceChangerManager.loadModel(configFilePath, pyTorchModelFilePath, onnxModelFilePath,
+                                                  hubertTorchModelFilePath, clusterTorchModelFilePath)
         json_compatible_item_data = jsonable_encoder(info)
         return JSONResponse(content=json_compatible_item_data)
         # return {"load": f"{configFilePath}, {pyTorchModelFilePath}, {onnxModelFilePath}"}

server/voice_changer/SoVitsSvc40v2/SoVitsSvc40v2.py

@@ -19,6 +19,7 @@ import onnxruntime
 import pyworld as pw
 
 from models import SynthesizerTrn
+import cluster
 import utils
 from fairseq import checkpoint_utils
 import librosa
@@ -36,6 +37,7 @@ class SoVitsSvc40v2Settings():
     predictF0: int = 0  # 0:False, 1:True
     silentThreshold: float = 0.00001
     extraConvertSize: int = 1024 * 32
+    clusterInferRatio: float = 0.1
 
     framework: str = "PyTorch"  # PyTorch or ONNX
     pyTorchModelFile: str = ""
@@ -44,7 +46,7 @@ class SoVitsSvc40v2Settings():
 
     # ↓mutableな物だけ列挙
     intData = ["gpu", "dstId", "tran", "predictF0", "extraConvertSize"]
-    floatData = ["noiceScale", "silentThreshold"]
+    floatData = ["noiceScale", "silentThreshold", "clusterInferRatio"]
     strData = ["framework", "f0Detector"]
 
 
@@ -58,20 +60,40 @@ class SoVitsSvc40v2:
         self.gpu_num = torch.cuda.device_count()
         self.prevVol = 0
 
-    def loadModel(self, config: str, pyTorch_model_file: str = None, onnx_model_file: str = None, hubertTorchModel: str = None):
+    def loadModel(self, config: str, pyTorch_model_file: str = None, onnx_model_file: str = None, hubertTorchModel: str = None, clusterTorchModel: str = None):
         self.settings.configFile = config
         self.hps = utils.get_hparams_from_file(config)
 
         # hubert model
-        # vec_path = "hubert/checkpoint_best_legacy_500.pt"
+        try:
-        vec_path = hubertTorchModel
+            vec_path = hubertTorchModel
-        models, saved_cfg, task = checkpoint_utils.load_model_ensemble_and_task(
+            print("hubert 1 ", hubertTorchModel)
-            [vec_path],
+            models, saved_cfg, task = checkpoint_utils.load_model_ensemble_and_task(
-            suffix="",
+                [vec_path],
-        )
+                suffix="",
-        model = models[0]
+            )
-        model.eval()
+            print("hubert 2 ", hubertTorchModel)
-        self.hubert_model = utils.get_hubert_model().cpu()
+            model = models[0]
+            print("hubert 3 ", hubertTorchModel)
+            model.eval()
+            print("hubert 4 ", hubertTorchModel)
+            self.hubert_model = model.cpu()
+            print("hubert 5 ", hubertTorchModel)
+        except Exception as e:
+            print("EXCEPTION1", e)
+
+        # cluster
+        try:
+            if os.path.exists(clusterTorchModel):
+                print("load kmean11", clusterTorchModel)
+                self.cluster_model = cluster.get_cluster_model(clusterTorchModel)
+                print("load kmean12", clusterTorchModel)
+            else:
+                print("load kmean21", clusterTorchModel)
+                self.cluster_model = None
+                print("load kmean22", clusterTorchModel)
+        except Exception as e:
+            print("EXCEPTION2", e)
 
         if pyTorch_model_file != None:
             self.settings.pyTorchModelFile = pyTorch_model_file
@@ -157,6 +179,14 @@ class SoVitsSvc40v2:
         wav16k = torch.from_numpy(wav16k)
         c = utils.get_hubert_content(self.hubert_model, wav_16k_tensor=wav16k)
         c = utils.repeat_expand_2d(c.squeeze(0), f0.shape[1])
+
+        if self.settings.clusterInferRatio != 0 and self.cluster_model != None:
+            # self.hsp.spk.tsukuyomi
+            cluster_c = cluster.get_cluster_center_result(self.cluster_model, c.cpu().numpy().T, "tsukuyomi").T
+            # cluster_c = cluster.get_cluster_center_result(self.cluster_model, c.cpu().numpy().T, self.settings.dstId).T
+            cluster_c = torch.FloatTensor(cluster_c).cpu()
+            c = self.settings.clusterInferRatio * cluster_c + (1 - self.settings.clusterInferRatio) * c
+
         c = c.unsqueeze(0)
         return c, f0, uv
 

server/voice_changer/VoiceChanger.py

@@ -70,11 +70,11 @@ class VoiceChanger():
 
         print(f"VoiceChanger Initialized (GPU_NUM:{self.gpu_num}, mps_enabled:{self.mps_enabled})")
 
-    def loadModel(self, config: str, pyTorch_model_file: str = None, onnx_model_file: str = None, hubertTorchModel: str = None):
+    def loadModel(self, config: str, pyTorch_model_file: str = None, onnx_model_file: str = None, hubertTorchModel: str = None, clusterTorchModel: str = None):
         if self.modelType == "MMVCv15" or self.modelType == "MMVCv13":
             return self.voiceChanger.loadModel(config, pyTorch_model_file, onnx_model_file)
         else:  # so-vits-svc-40v2
-            return self.voiceChanger.loadModel(config, pyTorch_model_file, onnx_model_file, hubertTorchModel)
+            return self.voiceChanger.loadModel(config, pyTorch_model_file, onnx_model_file, hubertTorchModel, clusterTorchModel)
 
     def get_info(self):
         data = asdict(self.settings)

server/voice_changer/VoiceChangerManager.py

@@ -10,8 +10,8 @@ class VoiceChangerManager():
             cls._instance.voiceChanger = VoiceChanger()
         return cls._instance
 
-    def loadModel(self, config, model, onnx_model, hubertTorchModel):
+    def loadModel(self, config, model, onnx_model, hubertTorchModel, clusterTorchModel):
-        info = self.voiceChanger.loadModel(config, model, onnx_model, hubertTorchModel)
+        info = self.voiceChanger.loadModel(config, model, onnx_model, hubertTorchModel, clusterTorchModel)
         info["status"] = "OK"
         return info