server audio

client/demo/dist/assets/gui_settings/RVC.json

@@ -68,7 +68,12 @@
                 "options": {}
             }
         ],
+
         "deviceSetting": [
+            {
+                "name": "audioDeviceMode",
+                "options": {}
+            },
             {
                 "name": "audioInput",
                 "options": {}
@@ -76,6 +81,10 @@
             {
                 "name": "audioOutput",
                 "options": {}
+            },
+            {
+                "name": "ioBuffer",
+                "options": {}
             }
         ],
         "qualityControl": [

client/demo/package.json

@@ -23,14 +23,14 @@
         "@babel/preset-env": "^7.21.5",
         "@babel/preset-react": "^7.18.6",
         "@babel/preset-typescript": "^7.21.5",
-        "@types/node": "^18.16.3",
+        "@types/node": "^20.1.0",
-        "@types/react": "^18.2.5",
+        "@types/react": "^18.2.6",
-        "@types/react-dom": "^18.2.3",
+        "@types/react-dom": "^18.2.4",
         "autoprefixer": "^10.4.14",
         "babel-loader": "^9.1.2",
         "copy-webpack-plugin": "^11.0.0",
         "css-loader": "^6.7.3",
-        "eslint": "^8.39.0",
+        "eslint": "^8.40.0",
         "eslint-config-prettier": "^8.8.0",
         "eslint-plugin-prettier": "^4.2.1",
         "eslint-plugin-react": "^7.32.2",
@@ -51,7 +51,7 @@
         "webpack-dev-server": "^4.13.3"
     },
     "dependencies": {
-        "@dannadori/voice-changer-client-js": "^1.0.122",
+        "@dannadori/voice-changer-client-js": "^1.0.123",
         "@fortawesome/fontawesome-svg-core": "^6.4.0",
         "@fortawesome/free-brands-svg-icons": "^6.4.0",
         "@fortawesome/free-regular-svg-icons": "^6.4.0",

client/demo/public/assets/gui_settings/RVC.json

@@ -68,7 +68,12 @@
                 "options": {}
             }
         ],
+
         "deviceSetting": [
+            {
+                "name": "audioDeviceMode",
+                "options": {}
+            },
             {
                 "name": "audioInput",
                 "options": {}
@@ -76,6 +81,10 @@
             {
                 "name": "audioOutput",
                 "options": {}
+            },
+            {
+                "name": "ioBuffer",
+                "options": {}
             }
         ],
         "qualityControl": [

client/demo/src/components/demo/002_ComponentGenerator.tsx

@@ -46,6 +46,8 @@ import { ONNXExecutorRow, ONNXExecutorRowProps } from "./components/206_ONNXExec
 import { MergeLabRow, MergeLabRowProps } from "./components/a01_MergeLab.Row"
 import { ModelSwitchRow, ModelSwitchRowProps } from "./components/204v2_ModelSwitchRow"
 import { EnableDirectMLRow, EnableDirectMLRowProps } from "./components/813_EnableDirectMLRow"
+import { AudioDeviceModeRow, AudioDeviceModeRowProps } from "./components/410_AudioDeviceModeRow"
+import { IOBufferRow, IOBufferRowProps } from "./components/411_IOBufferRow"
 
 export const catalog: { [key: string]: (props: any) => JSX.Element } = {}
 
@@ -83,6 +85,10 @@ const initialize = () => {
 
     addToCatalog("audioInput", (props: AudioInputRowProps) => { return <AudioInputRow {...props} /> })
     addToCatalog("audioOutput", (props: AudioOutputRowProps) => { return <AudioOutputRow {...props} /> })
+    addToCatalog("audioDeviceMode", (props: AudioDeviceModeRowProps) => { return <AudioDeviceModeRow {...props} /> })
+
+
+
 
     addToCatalog("noiseControl", (props: NoiseControlRowProps) => { return <NoiseControlRow {...props} /> })
     addToCatalog("gainControl", (props: GainControlRowProps) => { return <GainControlRow {...props} /> })
@@ -105,7 +111,8 @@ const initialize = () => {
 
     addToCatalog("inputChunkNum", (props: InputChunkNumRowProps) => { return <InputChunkNumRow {...props} /> })
     addToCatalog("extraDataLength", (props: ExtraDataLengthRowProps) => { return <ExtraDataLengthRow {...props} /> })
-    addToCatalog("gpu", (props: GPURowProps) => { return < GPURow {...props} /> })
+    addToCatalog("gpu", (props: GPURowProps) => { return <GPURow {...props} /> })
+    addToCatalog("ioBuffer", (props: IOBufferRowProps) => { return <IOBufferRow {...props} /> })
 
     addToCatalog("serverURL", (props: ServerURLRowProps) => { return <ServerURLRow {...props} /> })
     addToCatalog("protocol", (props: ProtocolRowProps) => { return <ProtocolRow {...props} /> })

client/demo/src/components/demo/components/201_StartButtonRow.tsx

@@ -22,6 +22,7 @@ export const StartButtonRow = (_props: StartButtonRowProps) => {
 
     const startButtonRow = useMemo(() => {
         const onStartClicked = async () => {
+            if (appState.serverSetting.serverSetting.enableServerAudio == 0) {
                 if (!appState.initializedRef.current) {
                     while (true) {
                         // console.log("wait 500ms")
@@ -38,10 +39,20 @@ export const StartButtonRow = (_props: StartButtonRowProps) => {
                     guiState.setIsConverting(true)
                     await appState.clientSetting.start()
                 }
+            } else {
+                appState.serverSetting.updateServerSettings({ ...appState.serverSetting.serverSetting, serverAudioStated: 1 })
+                guiState.setIsConverting(true)
+            }
         }
         const onStopClicked = async () => {
+            if (appState.serverSetting.serverSetting.enableServerAudio == 0) {
                 guiState.setIsConverting(false)
                 await appState.clientSetting.stop()
+            } else {
+                guiState.setIsConverting(false)
+                console.log("Stop clicked", appState.serverSetting.serverSetting)
+                appState.serverSetting.updateServerSettings({ ...appState.serverSetting.serverSetting, serverAudioStated: 0 })
+            }
         }
         const startClassName = guiState.isConverting ? "body-button-active" : "body-button-stanby"
         const stopClassName = guiState.isConverting ? "body-button-stanby" : "body-button-active"
@@ -59,7 +70,7 @@ export const StartButtonRow = (_props: StartButtonRowProps) => {
                 </div>
             </div>
         )
-    }, [guiState.isConverting, appState.clientSetting.start, appState.clientSetting.stop])
+    }, [guiState.isConverting, appState.clientSetting.start, appState.clientSetting.stop, appState.serverSetting.serverSetting, , appState.serverSetting.updateServerSettings])
 
     return startButtonRow
 }

client/demo/src/components/demo/components/202_PerformanceRow.tsx

@@ -1,4 +1,4 @@
-import React, { useMemo, useState } from "react"
+import React, { useEffect, useMemo, useState } from "react"
 import { useAppState } from "../../../001_provider/001_AppStateProvider"
 
 export type PerformanceRowProps = {
@@ -31,7 +31,31 @@ export const PerformanceRow = (_props: PerformanceRowProps) => {
                 </div>
             </>
         )
-    }, [appState.volume, appState.bufferingTime, appState.performance, showPerformanceDetail])
+    }, [appState.volume, appState.bufferingTime, appState.performance, showPerformanceDetail, appState.serverSetting.serverSetting.enableServerAudio])
 
-    return performanceRow
+    useEffect(() => {
+        if (!appState.updatePerformance) {
+            return
+        }
+        if (appState.serverSetting.serverSetting.enableServerAudio != 1) {
+            return
+        }
+        let execNext = true
+        const updatePerformance = async () => {
+            await appState.updatePerformance()
+            if (execNext) {
+                setTimeout(updatePerformance, 1000 * 2)
+            }
+        }
+        updatePerformance()
+        return () => {
+            execNext = false
+        }
+    }, [appState.updatePerformance, appState.serverSetting.serverSetting.enableServerAudio])
+
+    return (
+        <>
+            {performanceRow}
+        </>
+    )
 }

client/demo/src/components/demo/components/204v2_ModelSwitchRow.tsx

@@ -38,7 +38,7 @@ export const ModelSwitchRow = (_props: ModelSwitchRowProps) => {
             } else if (x.onnxModelFile && x.onnxModelFile.length > 0) {
                 filename = x.onnxModelFile.replace(/^.*[\\\/]/, '')
             } else {
-                return <div key={index} ></div>
+                return null
             }
             const f0str = x.f0 == true ? "f0" : "nof0"
             const srstr = Math.floor(x.samplingRate / 1000) + "K"
@@ -51,8 +51,8 @@ export const ModelSwitchRow = (_props: ModelSwitchRowProps) => {
             return (
                 <option key={index} value={index}>{displayName}</option>
             )
+        }).filter(x => { return x != null })
 
-        })
         return (
             <>
                 <div className="body-row split-3-7 left-padding-1 guided">
@@ -60,7 +60,6 @@ export const ModelSwitchRow = (_props: ModelSwitchRowProps) => {
                     <div className="body-input-container">
                         <select className="body-select" value={slot} onChange={(e) => {
                             onSwitchModelClicked(Number(e.target.value))
-                            appState.serverSetting.updateServerSettings({ ...appState.serverSetting.serverSetting, extraConvertSize: Number(e.target.value) })
                         }}>
                             {options}
                         </select>

client/demo/src/components/demo/components/401_AudioInputRow.tsx

@@ -1,4 +1,4 @@
-import React, { useMemo, useEffect } from "react"
+import React, { useMemo, useEffect, useState } from "react"
 import { useAppState } from "../../../001_provider/001_AppStateProvider"
 import { useGuiState } from "../001_GuiStateProvider"
 import { AudioInputMediaRow } from "./401-1_AudioInputMediaRow"
@@ -9,6 +9,7 @@ export type AudioInputRowProps = {
 export const AudioInputRow = (_props: AudioInputRowProps) => {
     const appState = useAppState()
     const guiState = useGuiState()
+    const [hostApi, setHostApi] = useState<string>("")
 
     // キャッシュの設定は反映（たぶん、設定操作の時も起動していしまう。が問題は起こらないはず）
     useEffect(() => {
@@ -24,6 +25,10 @@ export const AudioInputRow = (_props: AudioInputRowProps) => {
 
 
     const audioInputRow = useMemo(() => {
+        if (appState.serverSetting.serverSetting.enableServerAudio == 1) {
+            return <></>
+        }
+
         return (
             <div className="body-row split-3-7 left-padding-1  guided">
                 <div className="body-item-title left-padding-1">AudioInput</div>
@@ -43,7 +48,39 @@ export const AudioInputRow = (_props: AudioInputRowProps) => {
                 </div>
             </div>
         )
-    }, [guiState.inputAudioDeviceInfo, guiState.audioInputForGUI, appState.clientSetting.clientSetting, appState.clientSetting.updateClientSetting])
+    }, [guiState.inputAudioDeviceInfo, guiState.audioInputForGUI, appState.clientSetting.clientSetting, appState.clientSetting.updateClientSetting, appState.serverSetting.serverSetting.enableServerAudio])
+
+
+    const serverAudioInputRow = useMemo(() => {
+        if (appState.serverSetting.serverSetting.enableServerAudio == 0) {
+            return <></>
+        }
+        const devices = appState.serverSetting.serverSetting.serverAudioInputDevices
+        const hostAPIs = new Set(devices.map(x => { return x.hostAPI }))
+        const hostAPIOptions = Array.from(hostAPIs).map((x, index) => { return <option value={x} key={index} >{x}</option> })
+
+        const filteredDevice = devices.filter(x => { return x.hostAPI == hostApi || hostApi == "" }).map((x, index) => { return <option value={x.index} key={index}>{x.name}</option> })
+
+
+        return (
+            <div className="body-row split-3-7 left-padding-1  guided">
+                <div className="body-item-title left-padding-1">AudioInput</div>
+                <div className="body-select-container">
+                    <div className="body-select-container">
+                        <select name="kinds" id="kinds" value={hostApi} onChange={(e) => { setHostApi(e.target.value) }}>
+                            {hostAPIOptions}
+                        </select>
+                        <select className="body-select" value={appState.serverSetting.serverSetting.serverInputDeviceId} onChange={(e) => {
+                            appState.serverSetting.updateServerSettings({ ...appState.serverSetting.serverSetting, serverInputDeviceId: Number(e.target.value) })
+
+                        }}>
+                            {filteredDevice}
+                        </select>
+                    </div>
+                </div>
+            </div>
+        )
+    }, [hostApi, appState.serverSetting.serverSetting, appState.serverSetting.updateServerSettings])
 
 
 
@@ -51,6 +88,7 @@ export const AudioInputRow = (_props: AudioInputRowProps) => {
         <>
             {audioInputRow}
             <AudioInputMediaRow />
+            {serverAudioInputRow}
         </>
     )
 

client/demo/src/components/demo/components/402_AudioOutputRow.tsx

@@ -1,4 +1,4 @@
-import React, { useMemo, useEffect } from "react"
+import React, { useMemo, useEffect, useState } from "react"
 import { useIndexedDB } from "@dannadori/voice-changer-client-js"
 import { AudioOutputRecordRow } from "./402-1_AudioOutputRecordRow"
 import { useGuiState } from "../001_GuiStateProvider"
@@ -11,11 +11,12 @@ export type AudioOutputRowProps = {
 
 export const AudioOutputRow = (_props: AudioOutputRowProps) => {
     const { setAudioOutputElementId, initializedRef } = useAppState()
+    const appState = useAppState()
     const guiState = useGuiState()
     const { appGuiSettingState } = useAppRoot()
     const clientType = appGuiSettingState.appGuiSetting.id
     const { getItem, setItem } = useIndexedDB({ clientType: clientType })
-
+    const [hostApi, setHostApi] = useState<string>("")
 
     useEffect(() => {
         const loadCache = async () => {
@@ -34,7 +35,11 @@ export const AudioOutputRow = (_props: AudioOutputRowProps) => {
             [AUDIO_ELEMENT_FOR_PLAY_RESULT, AUDIO_ELEMENT_FOR_TEST_ORIGINAL, AUDIO_ELEMENT_FOR_TEST_CONVERTED_ECHOBACK].forEach(x => {
                 const audio = document.getElementById(x) as HTMLAudioElement
                 if (audio) {
-                    if (guiState.audioOutputForGUI == "none") {
+                    if (appState.serverSetting.serverSetting.enableServerAudio == 1) {
+
+                        // Server Audio を使う場合はElementから音は出さない。
+                        audio.volume = 0
+                    } else if (guiState.audioOutputForGUI == "none") {
                         // @ts-ignore
                         audio.setSinkId("")
                         if (x == AUDIO_ELEMENT_FOR_TEST_CONVERTED_ECHOBACK) {
@@ -51,6 +56,7 @@ export const AudioOutputRow = (_props: AudioOutputRowProps) => {
                         } else {
                             console.warn("No audio output device. use default")
                         }
+
                         if (x == AUDIO_ELEMENT_FOR_TEST_CONVERTED_ECHOBACK) {
                             audio.volume = guiState.fileInputEchoback ? 1 : 0
                         } else {
@@ -61,11 +67,15 @@ export const AudioOutputRow = (_props: AudioOutputRowProps) => {
             })
         }
         setAudioOutput()
-    }, [guiState.audioOutputForGUI, guiState.fileInputEchoback])
+    }, [guiState.audioOutputForGUI, guiState.fileInputEchoback, appState.serverSetting.serverSetting.enableServerAudio])
 
 
 
     const audioOutputRow = useMemo(() => {
+        if (appState.serverSetting.serverSetting.enableServerAudio == 1) {
+            return <></>
+        }
+
         return (
             <div className="body-row split-3-7 left-padding-1 guided">
                 <div className="body-item-title left-padding-1">AudioOutput</div>
@@ -80,20 +90,52 @@ export const AudioOutputRow = (_props: AudioOutputRowProps) => {
                             })
                         }
                     </select>
-                    <audio hidden id={AUDIO_ELEMENT_FOR_PLAY_RESULT}></audio>
                 </div>
             </div>
         )
-    }, [guiState.outputAudioDeviceInfo, guiState.audioOutputForGUI])
+    }, [appState.serverSetting.serverSetting.enableServerAudio, guiState.outputAudioDeviceInfo, guiState.audioOutputForGUI])
 
     useEffect(() => {
         setAudioOutputElementId(AUDIO_ELEMENT_FOR_PLAY_RESULT)
     }, [initializedRef.current])
 
+
+    const serverAudioOutputRow = useMemo(() => {
+        if (appState.serverSetting.serverSetting.enableServerAudio == 0) {
+            return <></>
+        }
+        const devices = appState.serverSetting.serverSetting.serverAudioOutputDevices
+        const hostAPIs = new Set(devices.map(x => { return x.hostAPI }))
+        const hostAPIOptions = Array.from(hostAPIs).map((x, index) => { return <option value={x} key={index} >{x}</option> })
+
+        const filteredDevice = devices.filter(x => { return x.hostAPI == hostApi || hostApi == "" }).map((x, index) => { return <option value={x.index} key={index}>{x.name}</option> })
+
+        return (
+            <div className="body-row split-3-7 left-padding-1  guided">
+                <div className="body-item-title left-padding-1">AudioOutput</div>
+                <div className="body-select-container">
+                    <div className="body-select-container">
+                        <select name="kinds" id="kinds" value={hostApi} onChange={(e) => { setHostApi(e.target.value) }}>
+                            {hostAPIOptions}
+                        </select>
+                        <select className="body-select" value={appState.serverSetting.serverSetting.serverOutputDeviceId} onChange={(e) => {
+                            appState.serverSetting.updateServerSettings({ ...appState.serverSetting.serverSetting, serverOutputDeviceId: Number(e.target.value) })
+                        }}>
+                            {filteredDevice}
+                        </select>
+                    </div>
+                </div>
+            </div>
+        )
+    }, [hostApi, appState.serverSetting.serverSetting, appState.serverSetting.updateServerSettings])
+
     return (
         <>
             {audioOutputRow}
-            <AudioOutputRecordRow />
+            {appState.serverSetting.serverSetting.enableServerAudio == 0 ? <AudioOutputRecordRow /> : <></>}
+
+            {serverAudioOutputRow}
+            <audio hidden id={AUDIO_ELEMENT_FOR_PLAY_RESULT}></audio>
         </>
     )
 }

client/demo/src/components/demo/components/410_AudioDeviceModeRow.tsx

@@ -0,0 +1,50 @@
+import React, { useMemo } from "react"
+import { useAppState } from "../../../001_provider/001_AppStateProvider"
+import { useGuiState } from "../001_GuiStateProvider"
+
+export type AudioDeviceModeRowProps = {
+}
+
+export const AudioDeviceModeRow = (_props: AudioDeviceModeRowProps) => {
+    const appState = useAppState()
+    const guiState = useGuiState()
+    const serverAudioInputRow = useMemo(() => {
+        const enableServerAudio = appState.serverSetting.serverSetting.enableServerAudio
+        const serverChecked = enableServerAudio == 1 ? true : false
+        const clientChecked = enableServerAudio == 1 ? false : true
+
+        const onDeviceModeChanged = (val: number) => {
+            if (guiState.isConverting) {
+                alert("cannot change mode when voice conversion is enabled")
+                return
+            }
+            appState.serverSetting.updateServerSettings({ ...appState.serverSetting.serverSetting, enableServerAudio: val })
+        }
+        return (
+            <div className="body-row split-3-3-4 left-padding-1  guided">
+
+                <div className="body-item-title left-padding-1">Device Mode</div>
+                <div className="body-input-container">
+                    <div className="left-padding-1">
+                        <input type="radio" id="client-device" name="device-mode" checked={clientChecked} onChange={() => { onDeviceModeChanged(0) }} />
+                        <label htmlFor="client-device">client device</label>
+                    </div>
+                    <div className="left-padding-1">
+                        <input className="left-padding-1" type="radio" id="server-device" name="device-mode" checked={serverChecked} onChange={() => { onDeviceModeChanged(1) }} />
+                        <label htmlFor="server-device">server device</label>
+                    </div>
+                </div>
+                <div></div>
+            </div>
+        )
+    }, [appState.serverSetting.serverSetting, appState.serverSetting.updateServerSettings, guiState.isConverting])
+
+
+
+    return (
+        <>
+            {serverAudioInputRow}
+        </>
+    )
+
+}

client/demo/src/components/demo/components/411_IOBufferRow.tsx

@@ -0,0 +1,56 @@
+import React, { useMemo } from "react"
+import { useAppState } from "../../../001_provider/001_AppStateProvider"
+
+export type IOBufferRowProps = {
+}
+export const IOBufferRow = (_props: IOBufferRowProps) => {
+    const appState = useAppState()
+
+    const ioBufferRow = useMemo(() => {
+        if (appState.serverSetting.serverSetting.enableServerAudio == 0) {
+            return <></>
+        }
+        const readBuf = appState.serverSetting.serverSetting.serverInputAudioBufferSize
+        const writeBuf = appState.serverSetting.serverSetting.serverOutputAudioBufferSize
+
+        return (
+            <div className="body-row split-3-3-4 left-padding-1 guided">
+                <div className="body-item-title left-padding-1">I/O Buffer</div>
+                <div className="body-input-container">
+                    <div className="left-padding-1">
+                        In:
+                        <select className="body-select" value={readBuf} onChange={(e) => {
+                            appState.serverSetting.updateServerSettings({ ...appState.serverSetting.serverSetting, serverInputAudioBufferSize: Number(e.target.value) })
+                            appState.workletNodeSetting.trancateBuffer()
+                        }}>
+                            {
+                                [1024 * 4, 1024 * 8, 1024 * 12, 1024 * 16, 1024 * 24, 1024 * 32].map(x => {
+                                    return <option key={x} value={x}>{x}</option>
+                                })
+                            }
+                        </select>
+                    </div>
+
+                    <div className="left-padding-1">
+                        Out:
+                        <select className="body-select" value={writeBuf} onChange={(e) => {
+                            appState.serverSetting.updateServerSettings({ ...appState.serverSetting.serverSetting, serverOutputAudioBufferSize: Number(e.target.value) })
+                            appState.workletNodeSetting.trancateBuffer()
+                        }}>
+                            {
+                                [1024 * 4, 1024 * 8, 1024 * 12, 1024 * 16, 1024 * 24, 1024 * 32].map(x => {
+                                    return <option key={x} value={x}>{x}</option>
+                                })
+                            }
+                        </select>
+
+                    </div>
+                </div>
+
+                <div className="body-item-text"></div>
+            </div>
+        )
+    }, [appState.serverSetting.serverSetting, appState.serverSetting.updateServerSettings])
+
+    return ioBufferRow
+}

client/demo/src/components/demo/components/701_InputChunkNumRow.tsx

@@ -6,6 +6,7 @@ export type InputChunkNumRowProps = {
 }
 export const InputChunkNumRow = (props: InputChunkNumRowProps) => {
     const appState = useAppState()
+
     const inputChunkNumRow = useMemo(() => {
         let nums: number[]
         if (!props.nums) {
@@ -20,6 +21,7 @@ export const InputChunkNumRow = (props: InputChunkNumRowProps) => {
                     <select className="body-select" value={appState.workletNodeSetting.workletNodeSetting.inputChunkNum} onChange={(e) => {
                         appState.workletNodeSetting.updateWorkletNodeSetting({ ...appState.workletNodeSetting.workletNodeSetting, inputChunkNum: Number(e.target.value) })
                         appState.workletNodeSetting.trancateBuffer()
+                        appState.serverSetting.updateServerSettings({ ...appState.serverSetting.serverSetting, serverReadChunkSize: Number(e.target.value) })
                     }}>
                         {
                             nums.map(x => {
@@ -35,7 +37,7 @@ export const InputChunkNumRow = (props: InputChunkNumRowProps) => {
 
             </div>
         )
-    }, [appState.workletNodeSetting.workletNodeSetting, appState.workletNodeSetting.updateWorkletNodeSetting])
+    }, [appState.workletNodeSetting.workletNodeSetting, appState.workletNodeSetting.updateWorkletNodeSetting, appState.serverSetting.serverSetting, appState.serverSetting.updateServerSettings])
 
     return inputChunkNumRow
 }

client/lib/package-lock.json

@@ -1,16 +1,16 @@
 {
   "name": "@dannadori/voice-changer-client-js",
-  "version": "1.0.122",
+  "version": "1.0.123",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "@dannadori/voice-changer-client-js",
-      "version": "1.0.122",
+      "version": "1.0.123",
       "license": "ISC",
       "dependencies": {
         "@types/readable-stream": "^2.3.15",
-        "amazon-chime-sdk-js": "^3.13.0",
+        "amazon-chime-sdk-js": "^3.14.0",
         "buffer": "^6.0.3",
         "localforage": "^1.10.0",
         "react": "^18.2.0",
@@ -19,10 +19,10 @@
       },
       "devDependencies": {
         "@types/audioworklet": "^0.0.45",
-        "@types/node": "^18.16.3",
+        "@types/node": "^20.1.0",
-        "@types/react": "18.2.5",
+        "@types/react": "18.2.6",
-        "@types/react-dom": "18.2.3",
+        "@types/react-dom": "18.2.4",
-        "eslint": "^8.39.0",
+        "eslint": "^8.40.0",
         "eslint-config-prettier": "^8.8.0",
         "eslint-plugin-prettier": "^4.2.1",
         "eslint-plugin-react": "^7.32.2",
@@ -1428,14 +1428,14 @@
       }
     },
     "node_modules/@eslint/eslintrc": {
-      "version": "2.0.2",
+      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.0.2.tgz",
+      "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.0.3.tgz",
-      "integrity": "sha512-3W4f5tDUra+pA+FzgugqL2pRimUTDJWKr7BINqOpkZrC0uYI0NIc0/JFgBROCU07HR6GieA5m3/rsPIhDmCXTQ==",
+      "integrity": "sha512-+5gy6OQfk+xx3q0d6jGZZC3f3KzAkXc/IanVxd1is/VIIziRqqt3ongQz0FiTUXqTk0c7aDB3OaFuKnuSoJicQ==",
       "dev": true,
       "dependencies": {
         "ajv": "^6.12.4",
         "debug": "^4.3.2",
-        "espree": "^9.5.1",
+        "espree": "^9.5.2",
         "globals": "^13.19.0",
         "ignore": "^5.2.0",
         "import-fresh": "^3.2.1",
@@ -1451,9 +1451,9 @@
       }
     },
     "node_modules/@eslint/js": {
-      "version": "8.39.0",
+      "version": "8.40.0",
-      "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.39.0.tgz",
+      "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.40.0.tgz",
-      "integrity": "sha512-kf9RB0Fg7NZfap83B3QOqOGg9QmD9yBudqQXzzOtn3i4y7ZUXe5ONeW34Gwi+TxhH4mvj72R1Zc300KUMa9Bng==",
+      "integrity": "sha512-ElyB54bJIhXQYVKjDSvCkPO1iU1tSAeVQJbllWJq1XQSmmA4dgFk8CbiBGpiOPxleE48vDogxCtmMYku4HSVLA==",
       "dev": true,
       "engines": {
         "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
@@ -1829,9 +1829,9 @@
       "dev": true
     },
     "node_modules/@types/node": {
-      "version": "18.16.3",
+      "version": "20.1.0",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-18.16.3.tgz",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.1.0.tgz",
-      "integrity": "sha512-OPs5WnnT1xkCBiuQrZA4+YAV4HEJejmHneyraIaxsbev5yCEr6KMwINNFP9wQeFIw8FWcoTqF3vQsa5CDaI+8Q=="
+      "integrity": "sha512-O+z53uwx64xY7D6roOi4+jApDGFg0qn6WHcxe5QeqjMaTezBO/mxdfFXIVAVVyNWKx84OmPB3L8kbVYOTeN34A=="
     },
     "node_modules/@types/prop-types": {
       "version": "15.7.5",
@@ -1852,9 +1852,9 @@
       "dev": true
     },
     "node_modules/@types/react": {
-      "version": "18.2.5",
+      "version": "18.2.6",
-      "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.5.tgz",
+      "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.6.tgz",
-      "integrity": "sha512-RuoMedzJ5AOh23Dvws13LU9jpZHIc/k90AgmK7CecAYeWmSr3553L4u5rk4sWAPBuQosfT7HmTfG4Rg5o4nGEA==",
+      "integrity": "sha512-wRZClXn//zxCFW+ye/D2qY65UsYP1Fpex2YXorHc8awoNamkMZSvBxwxdYVInsHOZZd2Ppq8isnSzJL5Mpf8OA==",
       "dev": true,
       "dependencies": {
         "@types/prop-types": "*",
@@ -1863,9 +1863,9 @@
       }
     },
     "node_modules/@types/react-dom": {
-      "version": "18.2.3",
+      "version": "18.2.4",
-      "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.2.3.tgz",
+      "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.2.4.tgz",
-      "integrity": "sha512-hxXEXWxFJXbY0LMj/T69mznqOZJXNtQMqVxIiirVAZnnpeYiD4zt+lPsgcr/cfWg2VLsxZ1y26vigG03prYB+Q==",
+      "integrity": "sha512-G2mHoTMTL4yoydITgOGwWdWMVd8sNgyEP85xVmMKAPUBwQWm9wBPQUmvbeF4V3WBY1P7mmL4BkjQ0SqUpf1snw==",
       "dev": true,
       "dependencies": {
         "@types/react": "*"
@@ -2259,9 +2259,9 @@
       }
     },
     "node_modules/amazon-chime-sdk-js": {
-      "version": "3.13.0",
+      "version": "3.14.0",
-      "resolved": "https://registry.npmjs.org/amazon-chime-sdk-js/-/amazon-chime-sdk-js-3.13.0.tgz",
+      "resolved": "https://registry.npmjs.org/amazon-chime-sdk-js/-/amazon-chime-sdk-js-3.14.0.tgz",
-      "integrity": "sha512-gRZ8SJgN2+0UMBRhUhAvc06ZklwDyYMEd/xJ17/lORX/XprcTVtRqicEJHxLtgzs61FKmmx1uBz0eGgrn9Dm3A==",
+      "integrity": "sha512-uNwCYygJahvncioPEzg4cDMPbYjpzbu1D92nUeYhUQcDK+qZn8zdPX8BRHI+eUSFBgI47udFPC933ZL79wXqJA==",
       "dependencies": {
         "@aws-crypto/sha256-js": "^2.0.1",
         "@aws-sdk/client-chime-sdk-messaging": "^3.0.0",
@@ -3230,15 +3230,15 @@
       }
     },
     "node_modules/eslint": {
-      "version": "8.39.0",
+      "version": "8.40.0",
-      "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.39.0.tgz",
+      "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.40.0.tgz",
-      "integrity": "sha512-mwiok6cy7KTW7rBpo05k6+p4YVZByLNjAZ/ACB9DRCu4YDRwjXI01tWHp6KAUWelsBetTxKK/2sHB0vdS8Z2Og==",
+      "integrity": "sha512-bvR+TsP9EHL3TqNtj9sCNJVAFK3fBN8Q7g5waghxyRsPLIMwL73XSKnZFK0hk/O2ANC+iAoq6PWMQ+IfBAJIiQ==",
       "dev": true,
       "dependencies": {
         "@eslint-community/eslint-utils": "^4.2.0",
         "@eslint-community/regexpp": "^4.4.0",
-        "@eslint/eslintrc": "^2.0.2",
+        "@eslint/eslintrc": "^2.0.3",
-        "@eslint/js": "8.39.0",
+        "@eslint/js": "8.40.0",
         "@humanwhocodes/config-array": "^0.11.8",
         "@humanwhocodes/module-importer": "^1.0.1",
         "@nodelib/fs.walk": "^1.2.8",
@@ -3249,8 +3249,8 @@
         "doctrine": "^3.0.0",
         "escape-string-regexp": "^4.0.0",
         "eslint-scope": "^7.2.0",
-        "eslint-visitor-keys": "^3.4.0",
+        "eslint-visitor-keys": "^3.4.1",
-        "espree": "^9.5.1",
+        "espree": "^9.5.2",
         "esquery": "^1.4.2",
         "esutils": "^2.0.2",
         "fast-deep-equal": "^3.1.3",
@@ -3377,9 +3377,9 @@
       }
     },
     "node_modules/eslint-visitor-keys": {
-      "version": "3.4.0",
+      "version": "3.4.1",
-      "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.0.tgz",
+      "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.1.tgz",
-      "integrity": "sha512-HPpKPUBQcAsZOsHAFwTtIKcYlCje62XB7SEAcxjtmW6TD1WVpkS6i6/hOVtTZIl4zGj/mBqpFVGvaDneik+VoQ==",
+      "integrity": "sha512-pZnmmLwYzf+kWaM/Qgrvpen51upAktaaiI01nsJD/Yr3lMOdNtq0cxkrrg16w64VtisN6okbs7Q8AfGqj4c9fA==",
       "dev": true,
       "engines": {
         "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
@@ -3413,14 +3413,14 @@
       }
     },
     "node_modules/espree": {
-      "version": "9.5.1",
+      "version": "9.5.2",
-      "resolved": "https://registry.npmjs.org/espree/-/espree-9.5.1.tgz",
+      "resolved": "https://registry.npmjs.org/espree/-/espree-9.5.2.tgz",
-      "integrity": "sha512-5yxtHSZXRSW5pvv3hAlXM5+/Oswi1AUFqBmbibKb5s6bp3rGIDkyXU6xCoyuuLhijr4SFwPrXRoZjz0AZDN9tg==",
+      "integrity": "sha512-7OASN1Wma5fum5SrNhFMAMJxOUAbhyfQ8dQ//PJaJbNw0URTPWqIghHWt1MmAANKhHZIYOHruW4Kw4ruUWOdGw==",
       "dev": true,
       "dependencies": {
         "acorn": "^8.8.0",
         "acorn-jsx": "^5.3.2",
-        "eslint-visitor-keys": "^3.4.0"
+        "eslint-visitor-keys": "^3.4.1"
       },
       "engines": {
         "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
@@ -9118,14 +9118,14 @@
       "dev": true
     },
     "@eslint/eslintrc": {
-      "version": "2.0.2",
+      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.0.2.tgz",
+      "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.0.3.tgz",
-      "integrity": "sha512-3W4f5tDUra+pA+FzgugqL2pRimUTDJWKr7BINqOpkZrC0uYI0NIc0/JFgBROCU07HR6GieA5m3/rsPIhDmCXTQ==",
+      "integrity": "sha512-+5gy6OQfk+xx3q0d6jGZZC3f3KzAkXc/IanVxd1is/VIIziRqqt3ongQz0FiTUXqTk0c7aDB3OaFuKnuSoJicQ==",
       "dev": true,
       "requires": {
         "ajv": "^6.12.4",
         "debug": "^4.3.2",
-        "espree": "^9.5.1",
+        "espree": "^9.5.2",
         "globals": "^13.19.0",
         "ignore": "^5.2.0",
         "import-fresh": "^3.2.1",
@@ -9135,9 +9135,9 @@
       }
     },
     "@eslint/js": {
-      "version": "8.39.0",
+      "version": "8.40.0",
-      "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.39.0.tgz",
+      "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.40.0.tgz",
-      "integrity": "sha512-kf9RB0Fg7NZfap83B3QOqOGg9QmD9yBudqQXzzOtn3i4y7ZUXe5ONeW34Gwi+TxhH4mvj72R1Zc300KUMa9Bng==",
+      "integrity": "sha512-ElyB54bJIhXQYVKjDSvCkPO1iU1tSAeVQJbllWJq1XQSmmA4dgFk8CbiBGpiOPxleE48vDogxCtmMYku4HSVLA==",
       "dev": true
     },
     "@humanwhocodes/config-array": {
@@ -9476,9 +9476,9 @@
       "dev": true
     },
     "@types/node": {
-      "version": "18.16.3",
+      "version": "20.1.0",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-18.16.3.tgz",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.1.0.tgz",
-      "integrity": "sha512-OPs5WnnT1xkCBiuQrZA4+YAV4HEJejmHneyraIaxsbev5yCEr6KMwINNFP9wQeFIw8FWcoTqF3vQsa5CDaI+8Q=="
+      "integrity": "sha512-O+z53uwx64xY7D6roOi4+jApDGFg0qn6WHcxe5QeqjMaTezBO/mxdfFXIVAVVyNWKx84OmPB3L8kbVYOTeN34A=="
     },
     "@types/prop-types": {
       "version": "15.7.5",
@@ -9499,9 +9499,9 @@
       "dev": true
     },
     "@types/react": {
-      "version": "18.2.5",
+      "version": "18.2.6",
-      "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.5.tgz",
+      "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.6.tgz",
-      "integrity": "sha512-RuoMedzJ5AOh23Dvws13LU9jpZHIc/k90AgmK7CecAYeWmSr3553L4u5rk4sWAPBuQosfT7HmTfG4Rg5o4nGEA==",
+      "integrity": "sha512-wRZClXn//zxCFW+ye/D2qY65UsYP1Fpex2YXorHc8awoNamkMZSvBxwxdYVInsHOZZd2Ppq8isnSzJL5Mpf8OA==",
       "dev": true,
       "requires": {
         "@types/prop-types": "*",
@@ -9510,9 +9510,9 @@
       }
     },
     "@types/react-dom": {
-      "version": "18.2.3",
+      "version": "18.2.4",
-      "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.2.3.tgz",
+      "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.2.4.tgz",
-      "integrity": "sha512-hxXEXWxFJXbY0LMj/T69mznqOZJXNtQMqVxIiirVAZnnpeYiD4zt+lPsgcr/cfWg2VLsxZ1y26vigG03prYB+Q==",
+      "integrity": "sha512-G2mHoTMTL4yoydITgOGwWdWMVd8sNgyEP85xVmMKAPUBwQWm9wBPQUmvbeF4V3WBY1P7mmL4BkjQ0SqUpf1snw==",
       "dev": true,
       "requires": {
         "@types/react": "*"
@@ -9854,9 +9854,9 @@
       "requires": {}
     },
     "amazon-chime-sdk-js": {
-      "version": "3.13.0",
+      "version": "3.14.0",
-      "resolved": "https://registry.npmjs.org/amazon-chime-sdk-js/-/amazon-chime-sdk-js-3.13.0.tgz",
+      "resolved": "https://registry.npmjs.org/amazon-chime-sdk-js/-/amazon-chime-sdk-js-3.14.0.tgz",
-      "integrity": "sha512-gRZ8SJgN2+0UMBRhUhAvc06ZklwDyYMEd/xJ17/lORX/XprcTVtRqicEJHxLtgzs61FKmmx1uBz0eGgrn9Dm3A==",
+      "integrity": "sha512-uNwCYygJahvncioPEzg4cDMPbYjpzbu1D92nUeYhUQcDK+qZn8zdPX8BRHI+eUSFBgI47udFPC933ZL79wXqJA==",
       "requires": {
         "@aws-crypto/sha256-js": "^2.0.1",
         "@aws-sdk/client-chime-sdk-messaging": "^3.0.0",
@@ -10566,15 +10566,15 @@
       "dev": true
     },
     "eslint": {
-      "version": "8.39.0",
+      "version": "8.40.0",
-      "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.39.0.tgz",
+      "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.40.0.tgz",
-      "integrity": "sha512-mwiok6cy7KTW7rBpo05k6+p4YVZByLNjAZ/ACB9DRCu4YDRwjXI01tWHp6KAUWelsBetTxKK/2sHB0vdS8Z2Og==",
+      "integrity": "sha512-bvR+TsP9EHL3TqNtj9sCNJVAFK3fBN8Q7g5waghxyRsPLIMwL73XSKnZFK0hk/O2ANC+iAoq6PWMQ+IfBAJIiQ==",
       "dev": true,
       "requires": {
         "@eslint-community/eslint-utils": "^4.2.0",
         "@eslint-community/regexpp": "^4.4.0",
-        "@eslint/eslintrc": "^2.0.2",
+        "@eslint/eslintrc": "^2.0.3",
-        "@eslint/js": "8.39.0",
+        "@eslint/js": "8.40.0",
         "@humanwhocodes/config-array": "^0.11.8",
         "@humanwhocodes/module-importer": "^1.0.1",
         "@nodelib/fs.walk": "^1.2.8",
@@ -10585,8 +10585,8 @@
         "doctrine": "^3.0.0",
         "escape-string-regexp": "^4.0.0",
         "eslint-scope": "^7.2.0",
-        "eslint-visitor-keys": "^3.4.0",
+        "eslint-visitor-keys": "^3.4.1",
-        "espree": "^9.5.1",
+        "espree": "^9.5.2",
         "esquery": "^1.4.2",
         "esutils": "^2.0.2",
         "fast-deep-equal": "^3.1.3",
@@ -10674,9 +10674,9 @@
       }
     },
     "eslint-visitor-keys": {
-      "version": "3.4.0",
+      "version": "3.4.1",
-      "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.0.tgz",
+      "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.1.tgz",
-      "integrity": "sha512-HPpKPUBQcAsZOsHAFwTtIKcYlCje62XB7SEAcxjtmW6TD1WVpkS6i6/hOVtTZIl4zGj/mBqpFVGvaDneik+VoQ==",
+      "integrity": "sha512-pZnmmLwYzf+kWaM/Qgrvpen51upAktaaiI01nsJD/Yr3lMOdNtq0cxkrrg16w64VtisN6okbs7Q8AfGqj4c9fA==",
       "dev": true
     },
     "eslint-webpack-plugin": {
@@ -10693,14 +10693,14 @@
       }
     },
     "espree": {
-      "version": "9.5.1",
+      "version": "9.5.2",
-      "resolved": "https://registry.npmjs.org/espree/-/espree-9.5.1.tgz",
+      "resolved": "https://registry.npmjs.org/espree/-/espree-9.5.2.tgz",
-      "integrity": "sha512-5yxtHSZXRSW5pvv3hAlXM5+/Oswi1AUFqBmbibKb5s6bp3rGIDkyXU6xCoyuuLhijr4SFwPrXRoZjz0AZDN9tg==",
+      "integrity": "sha512-7OASN1Wma5fum5SrNhFMAMJxOUAbhyfQ8dQ//PJaJbNw0URTPWqIghHWt1MmAANKhHZIYOHruW4Kw4ruUWOdGw==",
       "dev": true,
       "requires": {
         "acorn": "^8.8.0",
         "acorn-jsx": "^5.3.2",
-        "eslint-visitor-keys": "^3.4.0"
+        "eslint-visitor-keys": "^3.4.1"
       }
     },
     "esquery": {

client/lib/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dannadori/voice-changer-client-js",
-  "version": "1.0.122",
+  "version": "1.0.123",
   "description": "",
   "main": "dist/index.js",
   "directories": {
@@ -27,10 +27,10 @@
   "license": "ISC",
   "devDependencies": {
     "@types/audioworklet": "^0.0.45",
-    "@types/node": "^18.16.3",
+    "@types/node": "^20.1.0",
-    "@types/react": "18.2.5",
+    "@types/react": "18.2.6",
-    "@types/react-dom": "18.2.3",
+    "@types/react-dom": "18.2.4",
-    "eslint": "^8.39.0",
+    "eslint": "^8.40.0",
     "eslint-config-prettier": "^8.8.0",
     "eslint-plugin-prettier": "^4.2.1",
     "eslint-plugin-react": "^7.32.2",
@@ -47,7 +47,7 @@
   },
   "dependencies": {
     "@types/readable-stream": "^2.3.15",
-    "amazon-chime-sdk-js": "^3.13.0",
+    "amazon-chime-sdk-js": "^3.14.0",
     "buffer": "^6.0.3",
     "localforage": "^1.10.0",
     "react": "^18.2.0",

client/lib/src/ServerConfigurator.ts

@@ -27,6 +27,20 @@ export class ServerConfigurator {
         return info
     }
 
+    getPerformance = async () => {
+        const url = this.serverUrl + "/performance"
+        const info = await new Promise<number[]>((resolve) => {
+            const request = new Request(url, {
+                method: 'GET',
+            });
+            fetch(request).then(async (response) => {
+                const json = await response.json() as number[]
+                resolve(json)
+            })
+        })
+        return info
+    }
+
     updateSettings = async (key: ServerSettingKey, val: string) => {
         const url = this.serverUrl + "/update_settings"
         const info = await new Promise<ServerInfo>(async (resolve) => {

client/lib/src/VoiceChangerClient.ts

@@ -331,6 +331,9 @@ export class VoiceChangerClient {
     getServerSettings = () => {
         return this.configurator.getSettings()
     }
+    getPerformance = () => {
+        return this.configurator.getPerformance()
+    }
 
     getSocketId = () => {
         return this.vcInNode.getSocketId()

client/lib/src/VoiceChangerWorkletNode.ts

@@ -78,6 +78,7 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
                 console.log(`[SIO] ${this.socket?.id}`)
             });
             this.socket.on('response', (response: any[]) => {
+                console.log("response:", response)
                 const cur = Date.now()
                 const responseTime = cur - response[0]
                 const result = response[1] as ArrayBuffer

client/lib/src/const.ts

@@ -82,6 +82,16 @@ export const ServerSettingKey = {
     "f0Detector": "f0Detector",
     "recordIO": "recordIO",
 
+    "enableServerAudio": "enableServerAudio",
+    "serverAudioStated": "serverAudioStated",
+    "serverInputAudioSampleRate": "serverInputAudioSampleRate",
+    "serverOutputAudioSampleRate": "serverOutputAudioSampleRate",
+    "serverInputAudioBufferSize": "serverInputAudioBufferSize",
+    "serverOutputAudioBufferSize": "serverOutputAudioBufferSize",
+    "serverInputDeviceId": "serverInputDeviceId",
+    "serverOutputDeviceId": "serverOutputDeviceId",
+    "serverReadChunkSize": "serverReadChunkSize",
+
     "tran": "tran",
     "noiseScale": "noiseScale",
     "predictF0": "predictF0",
@@ -121,6 +131,17 @@ export type VoiceChangerServerSetting = {
     f0Detector: F0Detector // dio or harvest
     recordIO: number // 0:off, 1:on
 
+    enableServerAudio: number // 0:off, 1:on
+    serverAudioStated: number // 0:off, 1:on
+    serverInputAudioSampleRate: number
+    serverOutputAudioSampleRate: number
+    serverInputAudioBufferSize: number
+    serverOutputAudioBufferSize: number
+    serverInputDeviceId: number
+    serverOutputDeviceId: number
+    serverReadChunkSize: number
+
+
     tran: number // so-vits-svc
     noiseScale: number // so-vits-svc
     predictF0: number // so-vits-svc
@@ -156,6 +177,13 @@ type ModelSlot = {
     deprecated: boolean
 }
 
+type ServerAudioDevice = {
+    kind: "audioinput" | "audiooutput",
+    index: number,
+    name: string
+    hostAPI: string
+}
+
 export type ServerInfo = VoiceChangerServerSetting & {
     status: string
     configFile: string,
@@ -163,6 +191,9 @@ export type ServerInfo = VoiceChangerServerSetting & {
     onnxModelFile: string,
     onnxExecutionProviders: OnnxExecutionProvider[]
     modelSlots: ModelSlot[]
+    serverAudioInputDevices: ServerAudioDevice[]
+    serverAudioOutputDevices: ServerAudioDevice[]
+
 }
 
 export type ServerInfoSoVitsSVC = ServerInfo & {
@@ -179,6 +210,15 @@ export const DefaultServerSetting: ServerInfo = {
 
     recordIO: 0,
 
+    enableServerAudio: 0,
+    serverAudioStated: 0,
+    serverInputAudioSampleRate: 48000,
+    serverOutputAudioSampleRate: 48000,
+    serverInputAudioBufferSize: 1024 * 24,
+    serverOutputAudioBufferSize: 1024 * 24,
+    serverInputDeviceId: -1,
+    serverOutputDeviceId: -1,
+    serverReadChunkSize: 256,
 
     // VC Specific
     srcId: 0,
@@ -214,7 +254,9 @@ export const DefaultServerSetting: ServerInfo = {
     pyTorchModelFile: "",
     onnxModelFile: "",
     onnxExecutionProviders: [],
-    modelSlots: []
+    modelSlots: [],
+    serverAudioInputDevices: [],
+    serverAudioOutputDevices: []
 }
 export const DefaultServerSetting_MMVCv15: ServerInfo = {
     ...DefaultServerSetting, dstId: 101,

client/lib/src/hooks/useClient.ts

@@ -25,7 +25,7 @@ export type ClientState = {
     bufferingTime: number;
     volume: number;
     performance: PerformanceData
-
+    updatePerformance: (() => Promise<void>) | null
     // setClientType: (val: ClientType) => void
 
     // 情報取得
@@ -78,6 +78,31 @@ export const useClient = (props: UseClientProps): ClientState => {
     const [performance, setPerformance] = useState<PerformanceData>(InitialPerformanceData)
     const [volume, setVolume] = useState<number>(0)
 
+    //// Server Audio Deviceを使うとき、モニタリングデータはpolling
+    const updatePerformance = useMemo(() => {
+        if (!voiceChangerClientRef.current) {
+            return null
+        }
+
+        return async () => {
+            if (voiceChangerClientRef.current) {
+                const performance = await voiceChangerClientRef.current!.getPerformance()
+                const responseTime = performance[0]
+                const preprocessTime = performance[1]
+                const mainprocessTime = performance[2]
+                const postprocessTime = performance[3]
+                setPerformance({ responseTime, preprocessTime, mainprocessTime, postprocessTime })
+            } else {
+                const responseTime = 0
+                const preprocessTime = 0
+                const mainprocessTime = 0
+                const postprocessTime = 0
+                setPerformance({ responseTime, preprocessTime, mainprocessTime, postprocessTime })
+            }
+        }
+    }, [voiceChangerClientRef.current])
+
+
 
     // (1-4) エラーステータス
     const errorCountRef = useRef<number>(0)
@@ -168,6 +193,7 @@ export const useClient = (props: UseClientProps): ClientState => {
         bufferingTime,
         volume,
         performance,
+        updatePerformance,
 
         // setClientType,
 

client/lib/src/hooks/useServerSetting.ts

@@ -107,7 +107,11 @@ export const useServerSetting = (props: UseServerSettingProps): ServerSettingSta
             const cachedServerSetting = await getItem(INDEXEDDB_KEY_SERVER)
             let initialSetting: ServerInfo
             if (cachedServerSetting) {
-                initialSetting = { ...defaultServerSetting, ...cachedServerSetting as ServerInfo, inputSampleRate: 48000 }// sample rateは時限措置
+                initialSetting = {
+                    ...defaultServerSetting, ...cachedServerSetting as ServerInfo,
+                    serverAudioStated: 0,
+                    inputSampleRate: 48000
+                }// sample rateは時限措置
             } else {
                 initialSetting = { ...defaultServerSetting }
             }

server/const.py

@@ -90,3 +90,8 @@ class EnumPitchExtractorTypes(Enum):
 class EnumFrameworkTypes(Enum):
     pyTorch = "pyTorch"
     onnx = "onnx"
+
+
+class ServerAudioDeviceTypes(Enum):
+    audioinput = "audioinput"
+    audiooutput = "audiooutput"

server/restapi/MMVC_Rest_Fileuploader.py

@@ -23,6 +23,7 @@ class MMVC_Rest_Fileuploader:
         self.voiceChangerManager = voiceChangerManager
         self.router = APIRouter()
         self.router.add_api_route("/info", self.get_info, methods=["GET"])
+        self.router.add_api_route("/performance", self.get_performance, methods=["GET"])
         self.router.add_api_route(
             "/upload_file", self.post_upload_file, methods=["POST"]
         )
@@ -60,6 +61,11 @@ class MMVC_Rest_Fileuploader:
         json_compatible_item_data = jsonable_encoder(info)
         return JSONResponse(content=json_compatible_item_data)
 
+    def get_performance(self):
+        info = self.voiceChangerManager.get_performance()
+        json_compatible_item_data = jsonable_encoder(info)
+        return JSONResponse(content=json_compatible_item_data)
+
     def post_update_settings(
         self, key: str = Form(...), val: Union[int, str, float] = Form(...)
     ):

server/sio/MMVC_Namespace.py

@@ -6,6 +6,8 @@ from voice_changer.VoiceChangerManager import VoiceChangerManager
 
 
 class MMVC_Namespace(socketio.AsyncNamespace):
+    sid: int = 0
+
     def __init__(self, namespace: str, voiceChangerManager: VoiceChangerManager):
         super().__init__(namespace)
         self.voiceChangerManager = voiceChangerManager
@@ -17,24 +19,32 @@ class MMVC_Namespace(socketio.AsyncNamespace):
         return cls._instance
 
     def on_connect(self, sid, environ):
-        print('[{}] connet sid : {}'.format(datetime.now().strftime('%Y-%m-%d %H:%M:%S'), sid))
+        self.sid = sid
+        print(
+            "[{}] connet sid : {}".format(
+                datetime.now().strftime("%Y-%m-%d %H:%M:%S"), sid
+            )
+        )
         pass
 
     async def on_request_message(self, sid, msg):
+        self.sid = sid
         timestamp = int(msg[0])
         data = msg[1]
-        if (isinstance(data, str)):
+        if isinstance(data, str):
             print(type(data))
             print(data)
-            await self.emit('response', [timestamp, 0], to=sid)
+            await self.emit("response", [timestamp, 0], to=sid)
         else:
-            unpackedData = np.array(struct.unpack('<%sh' % (len(data) // struct.calcsize('<h')), data)).astype(np.int16)
+            unpackedData = np.array(
+                struct.unpack("<%sh" % (len(data) // struct.calcsize("<h")), data)
+            ).astype(np.int16)
 
             res = self.voiceChangerManager.changeVoice(unpackedData)
             audio1 = res[0]
             perf = res[1] if len(res) == 2 else [0, 0, 0]
-            bin = struct.pack('<%sh' % len(audio1), *audio1)
+            bin = struct.pack("<%sh" % len(audio1), *audio1)
-            await self.emit('response', [timestamp, bin, perf], to=sid)
+            await self.emit("response", [timestamp, bin, perf], to=sid)
 
     def on_disconnect(self, sid):
         # print('[{}] disconnect'.format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')))

server/voice_changer/Local/AudioDeviceList.py

@@ -0,0 +1,58 @@
+import pyaudio
+
+# import json
+
+
+from dataclasses import dataclass
+
+from const import ServerAudioDeviceTypes
+
+
+@dataclass
+class ServerAudioDevice:
+    kind: ServerAudioDeviceTypes = ServerAudioDeviceTypes.audioinput
+    index: int = 0
+    name: str = ""
+    hostAPI: str = ""
+
+
+def list_audio_device():
+    audio = pyaudio.PyAudio()
+    audio_input_devices: list[ServerAudioDevice] = []
+    audio_output_devices: list[ServerAudioDevice] = []
+    # audio_devices = {}
+    host_apis = []
+
+    for api_index in range(audio.get_host_api_count()):
+        host_apis.append(audio.get_host_api_info_by_index(api_index)["name"])
+
+    for x in range(0, audio.get_device_count()):
+        device = audio.get_device_info_by_index(x)
+        try:
+            deviceName = device["name"].encode("shift-jis").decode("utf-8")
+        except (UnicodeDecodeError, UnicodeEncodeError):
+            deviceName = device["name"]
+
+        deviceIndex = device["index"]
+        hostAPI = host_apis[device["hostApi"]]
+
+        if device["maxInputChannels"] > 0:
+            audio_input_devices.append(
+                ServerAudioDevice(
+                    kind=ServerAudioDeviceTypes.audioinput,
+                    index=deviceIndex,
+                    name=deviceName,
+                    hostAPI=hostAPI,
+                )
+            )
+        if device["maxOutputChannels"] > 0:
+            audio_output_devices.append(
+                ServerAudioDevice(
+                    kind=ServerAudioDeviceTypes.audiooutput,
+                    index=deviceIndex,
+                    name=deviceName,
+                    hostAPI=hostAPI,
+                )
+            )
+
+    return audio_input_devices, audio_output_devices

server/voice_changer/VoiceChanger.py

@@ -1,4 +1,6 @@
 from typing import Any, Union, cast
+
+import socketio
 from const import TMP_DIR, ModelType
 import torch
 import os
@@ -9,6 +11,7 @@ import resampy
 
 
 from voice_changer.IORecorder import IORecorder
+from voice_changer.Local.AudioDeviceList import ServerAudioDevice, list_audio_device
 from voice_changer.utils.LoadModelParams import LoadModelParams
 
 from voice_changer.utils.Timer import Timer
@@ -21,6 +24,10 @@ from Exceptions import (
     ONNXInputArgumentException,
 )
 from voice_changer.utils.VoiceChangerParams import VoiceChangerParams
+import pyaudio
+import threading
+import struct
+import time
 
 providers = [
     "OpenVINOExecutionProvider",
@@ -42,10 +49,38 @@ class VoiceChangerSettings:
     crossFadeOverlapSize: int = 4096
 
     recordIO: int = 0  # 0:off, 1:on
+    serverAudioInputDevices: list[ServerAudioDevice] = field(default_factory=lambda: [])
+    serverAudioOutputDevices: list[ServerAudioDevice] = field(
+        default_factory=lambda: []
+    )
+
+    enableServerAudio: int = 0  # 0:off, 1:on
+    serverAudioStated: int = 0  # 0:off, 1:on
+    serverInputAudioSampleRate: int = 48000
+    serverOutputAudioSampleRate: int = 48000
+    serverInputAudioBufferSize: int = 1024 * 24
+    serverOutputAudioBufferSize: int = 1024 * 24
+    serverInputDeviceId: int = -1
+    serverOutputDeviceId: int = -1
+    serverReadChunkSize: int = 256
+    performance: list[int] = field(default_factory=lambda: [0, 0, 0, 0])
 
     # ↓mutableな物だけ列挙
     intData: list[str] = field(
-        default_factory=lambda: ["inputSampleRate", "crossFadeOverlapSize", "recordIO"]
+        default_factory=lambda: [
+            "inputSampleRate",
+            "crossFadeOverlapSize",
+            "recordIO",
+            "enableServerAudio",
+            "serverAudioStated",
+            "serverInputAudioSampleRate",
+            "serverOutputAudioSampleRate",
+            "serverInputAudioBufferSize",
+            "serverOutputAudioBufferSize",
+            "serverInputDeviceId",
+            "serverOutputDeviceId",
+            "serverReadChunkSize",
+        ]
     )
     floatData: list[str] = field(
         default_factory=lambda: ["crossFadeOffsetRate", "crossFadeEndRate"]
@@ -53,11 +88,105 @@ class VoiceChangerSettings:
     strData: list[str] = field(default_factory=lambda: [])
 
 
+def serverLocal(_vc):
+    vc: VoiceChanger = _vc
+    audio = pyaudio.PyAudio()
+
+    def createAudioInput(deviceId: int, sampleRate: int, bufferSize: int):
+        audio_input_stream = audio.open(
+            format=pyaudio.paInt16,
+            channels=1,
+            rate=sampleRate,
+            # frames_per_buffer=32768,
+            frames_per_buffer=bufferSize,
+            input_device_index=deviceId,
+            input=True,
+        )
+        return audio_input_stream
+
+    def createAudioOutput(deviceId: int, sampleRate: int, bufferSize: int):
+        audio_output_stream = audio.open(
+            format=pyaudio.paInt16,
+            channels=1,
+            rate=sampleRate,
+            # frames_per_buffer=32768,
+            frames_per_buffer=bufferSize,
+            output_device_index=deviceId,
+            output=True,
+        )
+        return audio_output_stream
+
+    currentInputDeviceId = -1
+    currentInputSampleRate = -1
+    currentInputBufferSize = -1
+    currentOutputDeviceId = -1
+    currentOutputSampleRate = -1
+    currentOutputBufferSize = -1
+
+    audio_input_stream = None
+    audio_output_stream = None
+    while True:
+        if (
+            vc.settings.enableServerAudio == 0
+            or vc.settings.serverAudioStated == 0
+            or vc.settings.serverInputDeviceId == -1
+            or vc.settings.serverOutputDeviceId == -1
+        ):
+            time.sleep(2)
+        else:
+            if (
+                currentInputDeviceId != vc.settings.serverInputDeviceId
+                or currentInputSampleRate != vc.settings.serverInputAudioSampleRate
+                or currentInputBufferSize != vc.settings.serverInputAudioBufferSize
+            ):
+                currentInputDeviceId = vc.settings.serverInputDeviceId
+                currentInputSampleRate = vc.settings.serverInputAudioSampleRate
+                currentInputBufferSize = vc.settings.serverInputAudioBufferSize
+                if audio_input_stream is not None:
+                    audio_input_stream.close()
+                audio_input_stream = createAudioInput(
+                    currentInputDeviceId,
+                    currentInputSampleRate,
+                    currentInputBufferSize,
+                )
+
+            if (
+                currentOutputDeviceId != vc.settings.serverOutputDeviceId
+                or currentOutputSampleRate != vc.settings.serverOutputAudioSampleRate
+                or currentOutputBufferSize != vc.settings.serverOutputAudioBufferSize
+            ):
+                currentOutputDeviceId = vc.settings.serverOutputDeviceId
+                currentOutputSampleRate = vc.settings.serverOutputAudioSampleRate
+                currentOutputBufferSize = vc.settings.serverOutputAudioBufferSize
+                if audio_output_stream is not None:
+                    audio_output_stream.close()
+                audio_output_stream = createAudioOutput(
+                    currentOutputDeviceId,
+                    currentOutputSampleRate,
+                    currentOutputBufferSize,
+                )
+
+            in_wav = audio_input_stream.read(
+                vc.settings.serverReadChunkSize * 128, exception_on_overflow=False
+            )
+            unpackedData = np.array(
+                struct.unpack("<%sh" % (len(in_wav) // struct.calcsize("<h")), in_wav)
+            ).astype(np.int16)
+            with Timer("all_inference_time") as t:
+                out_wav, times = vc.on_request(unpackedData)
+            all_inference_time = t.secs
+            performance = [all_inference_time] + times
+            performance = [round(x, 2) * 1000 for x in performance]
+            vc.settings.performance = performance
+            audio_output_stream.write(out_wav.tobytes())
+
+
 class VoiceChanger:
     settings: VoiceChangerSettings
     voiceChanger: VoiceChangerModel
     ioRecorder: IORecorder
     sola_buffer: AudioInOut
+    namespace: socketio.AsyncNamespace | None = None
 
     def __init__(self, params: VoiceChangerParams):
         # 初期化
@@ -78,6 +207,12 @@ class VoiceChanger:
             and torch.backends.mps.is_available()
         )
 
+        audioinput, audiooutput = list_audio_device()
+        self.settings.serverAudioInputDevices = audioinput
+        self.settings.serverAudioOutputDevices = audiooutput
+
+        thread = threading.Thread(target=serverLocal, args=(self,))
+        thread.start()
         print(
             f"VoiceChanger Initialized (GPU_NUM:{self.gpu_num}, mps_enabled:{self.mps_enabled})"
         )
@@ -140,6 +275,9 @@ class VoiceChanger:
             data.update(self.voiceChanger.get_info())
         return data
 
+    def get_performance(self):
+        return self.settings.performance
+
     def update_settings(self, key: str, val: Any):
         if key in self.settings.intData:
             setattr(self.settings, key, int(val))

server/voice_changer/VoiceChangerManager.py

@@ -33,6 +33,13 @@ class VoiceChangerManager(object):
         else:
             return {"status": "ERROR", "msg": "no model loaded"}
 
+    def get_performance(self):
+        if hasattr(self, "voiceChanger"):
+            info = self.voiceChanger.get_performance()
+            return info
+        else:
+            return {"status": "ERROR", "msg": "no model loaded"}
+
     def update_settings(self, key: str, val: str | int | float):
         if hasattr(self, "voiceChanger"):
             info = self.voiceChanger.update_settings(key, val)