WIP: directml

client/demo/dist/assets/gui_settings/RVC.json

@@ -165,6 +165,10 @@
             {
                 "name": "silenceFront",
                 "options": {}
+            },
+            {
+                "name": "enableDirectML",
+                "options": {}
             }
         ]
     },

client/demo/public/assets/gui_settings/RVC.json

@@ -165,6 +165,10 @@
             {
                 "name": "silenceFront",
                 "options": {}
+            },
+            {
+                "name": "enableDirectML",
+                "options": {}
             }
         ]
     },

client/demo/src/components/demo/002_ComponentGenerator.tsx

@@ -45,6 +45,7 @@ import { ONNXExportRow, ONNXExportRowProps } from "./components/205_ONNXExportRo
 import { ONNXExecutorRow, ONNXExecutorRowProps } from "./components/206_ONNXExecutorRow"
 import { MergeLabRow, MergeLabRowProps } from "./components/a01_MergeLab.Row"
 import { ModelSwitchRow, ModelSwitchRowProps } from "./components/204v2_ModelSwitchRow"
+import { EnableDirectMLRow, EnableDirectMLRowProps } from "./components/813_EnableDirectMLRow"
 
 export const catalog: { [key: string]: (props: any) => JSX.Element } = {}
 
@@ -117,6 +118,9 @@ const initialize = () => {
     addToCatalog("trancateNumThreshold", (props: TrancateNumTresholdRowProps) => { return <TrancateNumTresholdRow {...props} /> })
     addToCatalog("rvcQuality", (props: RVCQualityRowProps) => { return <RVCQualityRow {...props} /> })
     addToCatalog("silenceFront", (props: SilenceFrontRowProps) => { return <SilenceFrontRow {...props} /> })
+    addToCatalog("enableDirectML", (props: EnableDirectMLRowProps) => { return <EnableDirectMLRow {...props} /> })
+
+
 
     addToCatalog("mergeLab", (props: MergeLabRowProps) => { return <MergeLabRow {...props} /> })
 

client/demo/src/components/demo/components/813_EnableDirectMLRow.tsx

@@ -0,0 +1,31 @@
+import React, { useMemo } from "react"
+import { useAppState } from "../../../001_provider/001_AppStateProvider"
+
+export type EnableDirectMLRowProps = {
+}
+
+export const EnableDirectMLRow = (_props: EnableDirectMLRowProps) => {
+    const appState = useAppState()
+
+    const enableDirctMLRow = useMemo(() => {
+        const onEnableDirectMLChanged = (val: number) => {
+            appState.serverSetting.updateServerSettings({
+                ...appState.serverSetting.serverSetting,
+                enableDirectML: val
+            })
+        }
+        return (
+            <div className="body-row split-3-7 left-padding-1 guided">
+                <div className="body-item-title left-padding-1">DirectML(experimental)</div>
+                <div className="body-input-container">
+                    <select value={appState.serverSetting.serverSetting.enableDirectML} onChange={(e) => { onEnableDirectMLChanged(Number(e.target.value)) }}>
+                        <option value="0" >off</option>
+                        <option value="1" >on</option>
+                    </select>
+                </div>
+            </div>
+        )
+    }, [appState.serverSetting.serverSetting, appState.serverSetting.updateServerSettings])
+
+    return enableDirctMLRow
+}

client/lib/src/const.ts

@@ -99,6 +99,8 @@ export const ServerSettingKey = {
     "enhancerTune": "enhancerTune",
 
     "inputSampleRate": "inputSampleRate",
+
+    "enableDirectML": "enableDirectML",
 } as const
 export type ServerSettingKey = typeof ServerSettingKey[keyof typeof ServerSettingKey]
 
@@ -136,6 +138,7 @@ export type VoiceChangerServerSetting = {
     enhancerTune: number // DDSP-SVC
 
     inputSampleRate: InputSampleRate
+    enableDirectML: number
 }
 
 type ModelSlot = {
@@ -204,7 +207,7 @@ export const DefaultServerSetting: ServerInfo = {
     enableEnhancer: 0,
     enhancerTune: 0,
 
-
+    enableDirectML: 0,
     // 
     status: "ok",
     configFile: "",

server/voice_changer/RVC/RVC.py

@@ -1,10 +1,12 @@
 import sys
 import os
-from Exceptions import NoModeLoadedException
+import json
-from voice_changer.RVC.ModelSlot import ModelSlot
+import resampy
-from voice_changer.RVC.deviceManager.DeviceManager import DeviceManager
+from dataclasses import asdict
+from typing import cast
+import numpy as np
+import torch
 
-from voice_changer.RVC.pitchExtractor.PitchExtractorManager import PitchExtractorManager
 
 # avoiding parse arg error in RVC
 sys.argv = ["MMVCServerSIO.py"]
@@ -18,34 +20,25 @@ if sys.platform.startswith("darwin"):
     sys.path.append(modulePath)
 else:
     sys.path.append("RVC")
-import json
+
-import resampy
 from voice_changer.RVC.modelMerger.MergeModel import merge_model
 from voice_changer.RVC.modelMerger.MergeModelRequest import MergeModelRequest
 from voice_changer.RVC.ModelSlotGenerator import generateModelSlot
 from voice_changer.RVC.RVCSettings import RVCSettings
 from voice_changer.RVC.embedder.EmbedderManager import EmbedderManager
-from voice_changer.RVC.inferencer.InferencerManager import InferencerManager
 from voice_changer.utils.LoadModelParams import FilePaths, LoadModelParams
 from voice_changer.utils.VoiceChangerModel import AudioInOut
 from voice_changer.utils.VoiceChangerParams import VoiceChangerParams
 from voice_changer.RVC.onnxExporter.export2onnx import export2onnx
+from voice_changer.RVC.pitchExtractor.PitchExtractorManager import PitchExtractorManager
+from voice_changer.RVC.pipeline.PipelineGenerator import createPipeline
+from voice_changer.RVC.deviceManager.DeviceManager import DeviceManager
+from voice_changer.RVC.pipeline.Pipeline import Pipeline
 
-from dataclasses import asdict
+from Exceptions import NoModeLoadedException
-from typing import cast
-import numpy as np
-import torch
-
-
-# from fairseq import checkpoint_utils
-import traceback
-import faiss
-
 from const import UPLOAD_DIR
 
 
-from voice_changer.RVC.Pipeline import Pipeline
-
 providers = [
     "OpenVINOExecutionProvider",
     "CUDAExecutionProvider",
@@ -96,121 +89,9 @@ class RVC:
             self.initialLoad = False
         elif target_slot_idx == self.currentSlot:
             self.prepareModel(target_slot_idx)
-            self.needSwitch = True
 
         return self.get_info()
 
-    def createPipeline(self, modelSlot: ModelSlot):
-        dev = self.deviceManager.getDevice(self.settings.gpu)
-        half = self.deviceManager.halfPrecisionAvailable(self.settings.gpu)
-        # ファイル名特定(Inferencer)
-        inferencerFilename = (
-            modelSlot.onnxModelFile if modelSlot.isONNX else modelSlot.pyTorchModelFile
-        )
-        
-        # Inferencer 生成
-        try:
-            inferencer = InferencerManager.getInferencer(
-                modelSlot.modelType,
-                inferencerFilename,
-                half,
-                dev,
-            )
-        except Exception as e:
-            print("[Voice Changer] exception! loading inferencer", e)
-            traceback.print_exc()
-
-        # Embedder 生成
-        try:
-            embedder = EmbedderManager.getEmbedder(
-                modelSlot.embedder,
-                # emmbedderFilename,
-                half,
-                dev,
-            )
-        except Exception as e:
-            print("[Voice Changer]  exception! loading embedder", e)
-            traceback.print_exc()
-
-        # pitchExtractor
-        pitchExtractor = PitchExtractorManager.getPitchExtractor(
-            self.settings.f0Detector
-        )
-
-        # index, feature
-        index, feature = self.loadIndex(modelSlot)
-
-        pipeline = Pipeline(
-            embedder,
-            inferencer,
-            pitchExtractor,
-            index,
-            feature,
-            modelSlot.samplingRate,
-            dev,
-            half,
-        )
-
-        return pipeline
-
-    def loadIndex(self, modelSlot: ModelSlot):
-        # Indexのロード
-        print("[Voice Changer] Loading index...")
-        # ファイル指定がない場合はNone
-        if modelSlot.featureFile is None or modelSlot.indexFile is None:
-            return None, None
-
-        # ファイル指定があってもファイルがない場合はNone
-        if (
-            os.path.exists(modelSlot.featureFile) is not True
-            or os.path.exists(modelSlot.indexFile) is not True
-        ):
-            return None, None
-
-        try:
-            index = faiss.read_index(modelSlot.indexFile)
-            feature = np.load(modelSlot.featureFile)
-        except:
-            print("[Voice Changer] load index failed. Use no index.")
-            traceback.print_exc()
-            return None, None
-
-        return index, feature
-
-    def prepareModel(self, slot: int):
-        if slot < 0:
-            return self.get_info()
-        modelSlot = self.settings.modelSlots[slot]
-        inferencerFilename = (
-            modelSlot.onnxModelFile if modelSlot.isONNX else modelSlot.pyTorchModelFile
-        )
-        if inferencerFilename == "":
-            return self.get_info()
-
-        print("[Voice Changer] Prepare Model of slot:", slot)
-
-        # pipelineの生成
-        self.next_pipeline = self.createPipeline(modelSlot)
-
-        # その他の設定
-        self.next_trans = modelSlot.defaultTrans
-        self.next_samplingRate = modelSlot.samplingRate
-        self.next_framework = "ONNX" if modelSlot.isONNX else "PyTorch"
-        self.needSwitch = True
-        print("[Voice Changer] Prepare done.")
-        return self.get_info()
-
-    def switchModel(self):
-        print("[Voice Changer] Switching model..")
-        self.pipeline = self.next_pipeline
-        self.settings.tran = self.next_trans
-        self.settings.modelSamplingRate = self.next_samplingRate
-        self.settings.framework = self.next_framework
-
-        print(
-            "[Voice Changer] Switching model..done",
-        )
-
     def update_settings(self, key: str, val: int | float | str):
         if key in self.settings.intData:
             # 設定前処理
@@ -238,6 +119,12 @@ class RVC:
                 else:
                     print("CHAGE TO NEW PIPELINE", half)
                     self.prepareModel(self.settings.modelSlotIndex)
+            if key == "enableDirectML":
+                if self.pipeline is not None and val == 0:
+                    self.pipeline.setDirectMLEnable(False)
+                elif self.pipeline is not None and val == 1:
+                    self.pipeline.setDirectMLEnable(True)
+
         elif key in self.settings.floatData:
             setattr(self.settings, key, float(val))
         elif key in self.settings.strData:
@@ -251,6 +138,42 @@ class RVC:
             return False
         return True
 
+    def prepareModel(self, slot: int):
+        if slot < 0:
+            return self.get_info()
+        modelSlot = self.settings.modelSlots[slot]
+        inferencerFilename = (
+            modelSlot.onnxModelFile if modelSlot.isONNX else modelSlot.pyTorchModelFile
+        )
+        if inferencerFilename == "":
+            return self.get_info()
+
+        print("[Voice Changer] Prepare Model of slot:", slot)
+
+        # pipelineの生成
+        self.next_pipeline = createPipeline(
+            modelSlot, self.settings.gpu, self.settings.f0Detector
+        )
+
+        # その他の設定
+        self.next_trans = modelSlot.defaultTrans
+        self.next_samplingRate = modelSlot.samplingRate
+        self.next_framework = "ONNX" if modelSlot.isONNX else "PyTorch"
+        self.needSwitch = True
+        print("[Voice Changer] Prepare done.")
+        return self.get_info()
+
+    def switchModel(self):
+        print("[Voice Changer] Switching model..")
+        self.pipeline = self.next_pipeline
+        self.settings.tran = self.next_trans
+        self.settings.modelSamplingRate = self.next_samplingRate
+        self.settings.framework = self.next_framework
+
+        print(
+            "[Voice Changer] Switching model..done",
+        )
+
     def get_info(self):
         data = asdict(self.settings)
         return data

server/voice_changer/RVC/RVCSettings.py

@@ -26,6 +26,7 @@ class RVCSettings:
 
     speakers: dict[str, int] = field(default_factory=lambda: {})
     isHalf: int = 1  # 0:off, 1:on
+    enableDirectML: int = 0  # 0:off, 1:on
     # ↓mutableな物だけ列挙
     intData = [
         "gpu",
@@ -37,6 +38,7 @@ class RVCSettings:
         "silenceFront",
         "modelSlotIndex",
         "isHalf",
+        "enableDirectML",
     ]
     floatData = ["silentThreshold", "indexRatio"]
     strData = ["framework", "f0Detector"]

server/voice_changer/RVC/inferencer/OnnxRVCInferencer.py

@@ -38,9 +38,9 @@ class OnnxRVCInferencer(Inferencer):
         if pitch is None or pitchf is None:
             raise RuntimeError("[Voice Changer] Pitch or Pitchf is not found.")
 
-        # print("INFER1", self.model.get_providers())
+        print("INFER1", self.model.get_providers())
-        # print("INFER2", self.model.get_provider_options())
+        print("INFER2", self.model.get_provider_options())
-        # print("INFER3", self.model.get_session_options())
+        print("INFER3", self.model.get_session_options())
         if self.isHalf:
             audio1 = self.model.run(
                 ["audio"],
@@ -86,3 +86,15 @@ class OnnxRVCInferencer(Inferencer):
             self.model.set_providers(providers=["CPUExecutionProvider"])
 
         return self
+
+    def setDirectMLEnable(self, enable: bool):
+        if "DmlExecutionProvider" not in onnxruntime.get_available_providers():
+            print("[Voice Changer] DML is not available.")
+            return
+
+        if enable:
+            self.model.set_providers(
+                providers=["DmlExecutionProvider", "CPUExecutionProvider"]
+            )
+        else:
+            self.model.set_providers(providers=["CPUExecutionProvider"])

server/voice_changer/RVC/pipeline/Pipeline.py

@@ -14,6 +14,10 @@ from voice_changer.RVC.inferencer.Inferencer import Inferencer
 from voice_changer.RVC.pitchExtractor.PitchExtractor import PitchExtractor
 
 
+# isHalfが変わる場合はPipeline作り直し
+# device(GPU, isHalf変更が伴わない場合), pitchExtractorの変更は、入れ替えで対応
+
+
 class Pipeline(object):
     embedder: Embedder
     inferencer: Inferencer
@@ -59,6 +63,10 @@ class Pipeline(object):
         self.embedder.setDevice(device)
         self.inferencer.setDevice(device)
 
+    def setDirectMLEnable(self, enable: bool):
+        if hasattr(self.inferencer, "setDirectMLEnable"):
+            self.inferencer.setDirectMLEnable(enable)
+
     def setPitchExtractor(self, pitchExtractor: PitchExtractor):
         self.pitchExtractor = pitchExtractor
 

server/voice_changer/RVC/pipeline/PipelineGenerator.py

@@ -0,0 +1,88 @@
+import os
+import traceback
+import numpy as np
+import faiss
+
+from voice_changer.RVC.ModelSlot import ModelSlot
+from voice_changer.RVC.deviceManager.DeviceManager import DeviceManager
+from voice_changer.RVC.embedder.EmbedderManager import EmbedderManager
+from voice_changer.RVC.inferencer.InferencerManager import InferencerManager
+from voice_changer.RVC.pipeline.Pipeline import Pipeline
+from voice_changer.RVC.pitchExtractor.PitchExtractorManager import PitchExtractorManager
+
+
+def createPipeline(modelSlot: ModelSlot, gpu: int, f0Detector: str):
+    dev = DeviceManager.get_instance().getDevice(gpu)
+    half = DeviceManager.get_instance().halfPrecisionAvailable(gpu)
+    # ファイル名特定(Inferencer)
+    inferencerFilename = (
+        modelSlot.onnxModelFile if modelSlot.isONNX else modelSlot.pyTorchModelFile
+    )
+
+    # Inferencer 生成
+    try:
+        inferencer = InferencerManager.getInferencer(
+            modelSlot.modelType,
+            inferencerFilename,
+            half,
+            dev,
+        )
+    except Exception as e:
+        print("[Voice Changer] exception! loading inferencer", e)
+        traceback.print_exc()
+
+    # Embedder 生成
+    try:
+        embedder = EmbedderManager.getEmbedder(
+            modelSlot.embedder,
+            # emmbedderFilename,
+            half,
+            dev,
+        )
+    except Exception as e:
+        print("[Voice Changer]  exception! loading embedder", e)
+        traceback.print_exc()
+
+    # pitchExtractor
+    pitchExtractor = PitchExtractorManager.getPitchExtractor(f0Detector)
+
+    # index, feature
+    index, feature = _loadIndex(modelSlot)
+
+    pipeline = Pipeline(
+        embedder,
+        inferencer,
+        pitchExtractor,
+        index,
+        feature,
+        modelSlot.samplingRate,
+        dev,
+        half,
+    )
+
+    return pipeline
+
+
+def _loadIndex(modelSlot: ModelSlot):
+    # Indexのロード
+    print("[Voice Changer] Loading index...")
+    # ファイル指定がない場合はNone
+    if modelSlot.featureFile is None or modelSlot.indexFile is None:
+        return None, None
+
+    # ファイル指定があってもファイルがない場合はNone
+    if (
+        os.path.exists(modelSlot.featureFile) is not True
+        or os.path.exists(modelSlot.indexFile) is not True
+    ):
+        return None, None
+
+    try:
+        index = faiss.read_index(modelSlot.indexFile)
+        feature = np.load(modelSlot.featureFile)
+    except:
+        print("[Voice Changer] load index failed. Use no index.")
+        traceback.print_exc()
+        return None, None
+
+    return index, feature