PublicMirrors/voice-changer
1
0
Fork 0
mirror of https://github.com/w-okada/voice-changer.git synced 2025-01-23 05:25:01 +03:00
Code Issues Actions Packages Projects Releases Wiki Activity

server audio

Browse Source
This commit is contained in:
wataru 2023-05-07 04:18:18 +09:00
parent f4e409187e
commit af4cf4857e
29 changed files with 1092 additions and 513 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
client/demo/dist/assets/gui_settings/RVC.json vendored
View File

@ -68,7 +68,12 @@
"options": {}
}
],
"deviceSetting": [
{
"name": "audioDeviceMode",
"options": {}
},
{
"name": "audioInput",
"options": {}
@ -76,6 +81,10 @@
{
"name": "audioOutput",
"options": {}
},
{
"name": "ioBuffer",
"options": {}
}
],
"qualityControl": [

2
client/demo/dist/index.js vendored
View File

File diff suppressed because one or more lines are too long

792
client/demo/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

10
client/demo/package.json
View File

@ -23,14 +23,14 @@
"@babel/preset-env": "^7.21.5",
"@babel/preset-react": "^7.18.6",
"@babel/preset-typescript": "^7.21.5",
"@types/node": "^18.16.3",
"@types/react": "^18.2.5",
"@types/react-dom": "^18.2.3",
"@types/node": "^20.1.0",
"@types/react": "^18.2.6",
"@types/react-dom": "^18.2.4",
"autoprefixer": "^10.4.14",
"babel-loader": "^9.1.2",
"copy-webpack-plugin": "^11.0.0",
"css-loader": "^6.7.3",
"eslint": "^8.39.0",
"eslint": "^8.40.0",
"eslint-config-prettier": "^8.8.0",
"eslint-plugin-prettier": "^4.2.1",
"eslint-plugin-react": "^7.32.2",
@ -51,7 +51,7 @@
"webpack-dev-server": "^4.13.3"
},
"dependencies": {
"@dannadori/voice-changer-client-js": "^1.0.122",
"@dannadori/voice-changer-client-js": "^1.0.123",
"@fortawesome/fontawesome-svg-core": "^6.4.0",
"@fortawesome/free-brands-svg-icons": "^6.4.0",
"@fortawesome/free-regular-svg-icons": "^6.4.0",

9
client/demo/public/assets/gui_settings/RVC.json
View File

@ -68,7 +68,12 @@
"options": {}
}
],
"deviceSetting": [
{
"name": "audioDeviceMode",
"options": {}
},
{
"name": "audioInput",
"options": {}
@ -76,6 +81,10 @@
{
"name": "audioOutput",
"options": {}
},
{
"name": "ioBuffer",
"options": {}
}
],
"qualityControl": [

9
client/demo/src/components/demo/002_ComponentGenerator.tsx
View File

@ -46,6 +46,8 @@ import { ONNXExecutorRow, ONNXExecutorRowProps } from "./components/206_ONNXExec
import { MergeLabRow, MergeLabRowProps } from "./components/a01_MergeLab.Row"
import { ModelSwitchRow, ModelSwitchRowProps } from "./components/204v2_ModelSwitchRow"
import { EnableDirectMLRow, EnableDirectMLRowProps } from "./components/813_EnableDirectMLRow"
import { AudioDeviceModeRow, AudioDeviceModeRowProps } from "./components/410_AudioDeviceModeRow"
import { IOBufferRow, IOBufferRowProps } from "./components/411_IOBufferRow"
export const catalog: { [key: string]: (props: any) => JSX.Element } = {}
@ -83,6 +85,10 @@ const initialize = () => {
addToCatalog("audioInput", (props: AudioInputRowProps) => { return <AudioInputRow {...props} /> })
addToCatalog("audioOutput", (props: AudioOutputRowProps) => { return <AudioOutputRow {...props} /> })
addToCatalog("audioDeviceMode", (props: AudioDeviceModeRowProps) => { return <AudioDeviceModeRow {...props} /> })
addToCatalog("noiseControl", (props: NoiseControlRowProps) => { return <NoiseControlRow {...props} /> })
addToCatalog("gainControl", (props: GainControlRowProps) => { return <GainControlRow {...props} /> })
@ -105,7 +111,8 @@ const initialize = () => {
addToCatalog("inputChunkNum", (props: InputChunkNumRowProps) => { return <InputChunkNumRow {...props} /> })
addToCatalog("extraDataLength", (props: ExtraDataLengthRowProps) => { return <ExtraDataLengthRow {...props} /> })
addToCatalog("gpu", (props: GPURowProps) => { return < GPURow {...props} /> })
addToCatalog("gpu", (props: GPURowProps) => { return <GPURow {...props} /> })
addToCatalog("ioBuffer", (props: IOBufferRowProps) => { return <IOBufferRow {...props} /> })
addToCatalog("serverURL", (props: ServerURLRowProps) => { return <ServerURLRow {...props} /> })
addToCatalog("protocol", (props: ProtocolRowProps) => { return <ProtocolRow {...props} /> })

13
client/demo/src/components/demo/components/201_StartButtonRow.tsx
View File

@ -22,6 +22,7 @@ export const StartButtonRow = (_props: StartButtonRowProps) => {
const startButtonRow = useMemo(() => {
const onStartClicked = async () => {
if (appState.serverSetting.serverSetting.enableServerAudio == 0) {
if (!appState.initializedRef.current) {
while (true) {
// console.log("wait 500ms")
@ -38,10 +39,20 @@ export const StartButtonRow = (_props: StartButtonRowProps) => {
guiState.setIsConverting(true)
await appState.clientSetting.start()
}
} else {
appState.serverSetting.updateServerSettings({ ...appState.serverSetting.serverSetting, serverAudioStated: 1 })
guiState.setIsConverting(true)
}
}
const onStopClicked = async () => {
if (appState.serverSetting.serverSetting.enableServerAudio == 0) {
guiState.setIsConverting(false)
await appState.clientSetting.stop()
} else {
guiState.setIsConverting(false)
console.log("Stop clicked", appState.serverSetting.serverSetting)
appState.serverSetting.updateServerSettings({ ...appState.serverSetting.serverSetting, serverAudioStated: 0 })
}
}
const startClassName = guiState.isConverting ? "body-button-active" : "body-button-stanby"
const stopClassName = guiState.isConverting ? "body-button-stanby" : "body-button-active"
@ -59,7 +70,7 @@ export const StartButtonRow = (_props: StartButtonRowProps) => {
</div>
</div>
)
}, [guiState.isConverting, appState.clientSetting.start, appState.clientSetting.stop])
}, [guiState.isConverting, appState.clientSetting.start, appState.clientSetting.stop, appState.serverSetting.serverSetting, , appState.serverSetting.updateServerSettings])
return startButtonRow
}

30
client/demo/src/components/demo/components/202_PerformanceRow.tsx
View File

@ -1,4 +1,4 @@
import React, { useMemo, useState } from "react"
import React, { useEffect, useMemo, useState } from "react"
import { useAppState } from "../../../001_provider/001_AppStateProvider"
export type PerformanceRowProps = {
@ -31,7 +31,31 @@ export const PerformanceRow = (_props: PerformanceRowProps) => {
</div>
</>
)
}, [appState.volume, appState.bufferingTime, appState.performance, showPerformanceDetail])
}, [appState.volume, appState.bufferingTime, appState.performance, showPerformanceDetail, appState.serverSetting.serverSetting.enableServerAudio])
return performanceRow
useEffect(() => {
if (!appState.updatePerformance) {
return
}
if (appState.serverSetting.serverSetting.enableServerAudio != 1) {
return
}
let execNext = true
const updatePerformance = async () => {
await appState.updatePerformance()
if (execNext) {
setTimeout(updatePerformance, 1000 * 2)
}
}
updatePerformance()
return () => {
execNext = false
}
}, [appState.updatePerformance, appState.serverSetting.serverSetting.enableServerAudio])
return (
<>
{performanceRow}
</>
)
}

5
client/demo/src/components/demo/components/204v2_ModelSwitchRow.tsx
View File

@ -38,7 +38,7 @@ export const ModelSwitchRow = (_props: ModelSwitchRowProps) => {
} else if (x.onnxModelFile && x.onnxModelFile.length > 0) {
filename = x.onnxModelFile.replace(/^.*[\\\/]/, '')
} else {
return <div key={index} ></div>
return null
}
const f0str = x.f0 == true ? "f0" : "nof0"
const srstr = Math.floor(x.samplingRate / 1000) + "K"
@ -51,8 +51,8 @@ export const ModelSwitchRow = (_props: ModelSwitchRowProps) => {
return (
<option key={index} value={index}>{displayName}</option>
)
}).filter(x => { return x != null })
})
return (
<>
<div className="body-row split-3-7 left-padding-1 guided">
@ -60,7 +60,6 @@ export const ModelSwitchRow = (_props: ModelSwitchRowProps) => {
<div className="body-input-container">
<select className="body-select" value={slot} onChange={(e) => {
onSwitchModelClicked(Number(e.target.value))
appState.serverSetting.updateServerSettings({ ...appState.serverSetting.serverSetting, extraConvertSize: Number(e.target.value) })
}}>
{options}
</select>

42
client/demo/src/components/demo/components/401_AudioInputRow.tsx
View File

@ -1,4 +1,4 @@
import React, { useMemo, useEffect } from "react"
import React, { useMemo, useEffect, useState } from "react"
import { useAppState } from "../../../001_provider/001_AppStateProvider"
import { useGuiState } from "../001_GuiStateProvider"
import { AudioInputMediaRow } from "./401-1_AudioInputMediaRow"
@ -9,6 +9,7 @@ export type AudioInputRowProps = {
export const AudioInputRow = (_props: AudioInputRowProps) => {
const appState = useAppState()
const guiState = useGuiState()
const [hostApi, setHostApi] = useState<string>("")
// キャッシュの設定は反映（たぶん、設定操作の時も起動していしまう。が問題は起こらないはず）
useEffect(() => {
@ -24,6 +25,10 @@ export const AudioInputRow = (_props: AudioInputRowProps) => {
const audioInputRow = useMemo(() => {
if (appState.serverSetting.serverSetting.enableServerAudio == 1) {
return <></>
}
return (
<div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1">AudioInput</div>
@ -43,7 +48,39 @@ export const AudioInputRow = (_props: AudioInputRowProps) => {
</div>
</div>
)
}, [guiState.inputAudioDeviceInfo, guiState.audioInputForGUI, appState.clientSetting.clientSetting, appState.clientSetting.updateClientSetting])
}, [guiState.inputAudioDeviceInfo, guiState.audioInputForGUI, appState.clientSetting.clientSetting, appState.clientSetting.updateClientSetting, appState.serverSetting.serverSetting.enableServerAudio])
const serverAudioInputRow = useMemo(() => {
if (appState.serverSetting.serverSetting.enableServerAudio == 0) {
return <></>
}
const devices = appState.serverSetting.serverSetting.serverAudioInputDevices
const hostAPIs = new Set(devices.map(x => { return x.hostAPI }))
const hostAPIOptions = Array.from(hostAPIs).map((x, index) => { return <option value={x} key={index} >{x}</option> })
const filteredDevice = devices.filter(x => { return x.hostAPI == hostApi || hostApi == "" }).map((x, index) => { return <option value={x.index} key={index}>{x.name}</option> })
return (
<div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1">AudioInput</div>
<div className="body-select-container">
<div className="body-select-container">
<select name="kinds" id="kinds" value={hostApi} onChange={(e) => { setHostApi(e.target.value) }}>
{hostAPIOptions}
</select>
<select className="body-select" value={appState.serverSetting.serverSetting.serverInputDeviceId} onChange={(e) => {
appState.serverSetting.updateServerSettings({ ...appState.serverSetting.serverSetting, serverInputDeviceId: Number(e.target.value) })
}}>
{filteredDevice}
</select>
</div>
</div>
</div>
)
}, [hostApi, appState.serverSetting.serverSetting, appState.serverSetting.updateServerSettings])
@ -51,6 +88,7 @@ export const AudioInputRow = (_props: AudioInputRowProps) => {
<>
{audioInputRow}
<AudioInputMediaRow />
{serverAudioInputRow}
</>
)

56
client/demo/src/components/demo/components/402_AudioOutputRow.tsx
View File

@ -1,4 +1,4 @@
import React, { useMemo, useEffect } from "react"
import React, { useMemo, useEffect, useState } from "react"
import { useIndexedDB } from "@dannadori/voice-changer-client-js"
import { AudioOutputRecordRow } from "./402-1_AudioOutputRecordRow"
import { useGuiState } from "../001_GuiStateProvider"
@ -11,11 +11,12 @@ export type AudioOutputRowProps = {
export const AudioOutputRow = (_props: AudioOutputRowProps) => {
const { setAudioOutputElementId, initializedRef } = useAppState()
const appState = useAppState()
const guiState = useGuiState()
const { appGuiSettingState } = useAppRoot()
const clientType = appGuiSettingState.appGuiSetting.id
const { getItem, setItem } = useIndexedDB({ clientType: clientType })
const [hostApi, setHostApi] = useState<string>("")
useEffect(() => {
const loadCache = async () => {
@ -34,7 +35,11 @@ export const AudioOutputRow = (_props: AudioOutputRowProps) => {
[AUDIO_ELEMENT_FOR_PLAY_RESULT, AUDIO_ELEMENT_FOR_TEST_ORIGINAL, AUDIO_ELEMENT_FOR_TEST_CONVERTED_ECHOBACK].forEach(x => {
const audio = document.getElementById(x) as HTMLAudioElement
if (audio) {
if (guiState.audioOutputForGUI == "none") {
if (appState.serverSetting.serverSetting.enableServerAudio == 1) {
// Server Audio を使う場合はElementから音は出さない。
audio.volume = 0
} else if (guiState.audioOutputForGUI == "none") {
// @ts-ignore
audio.setSinkId("")
if (x == AUDIO_ELEMENT_FOR_TEST_CONVERTED_ECHOBACK) {
@ -51,6 +56,7 @@ export const AudioOutputRow = (_props: AudioOutputRowProps) => {
} else {
console.warn("No audio output device. use default")
}
if (x == AUDIO_ELEMENT_FOR_TEST_CONVERTED_ECHOBACK) {
audio.volume = guiState.fileInputEchoback ? 1 : 0
} else {
@ -61,11 +67,15 @@ export const AudioOutputRow = (_props: AudioOutputRowProps) => {
})
}
setAudioOutput()
}, [guiState.audioOutputForGUI, guiState.fileInputEchoback])
}, [guiState.audioOutputForGUI, guiState.fileInputEchoback, appState.serverSetting.serverSetting.enableServerAudio])
const audioOutputRow = useMemo(() => {
if (appState.serverSetting.serverSetting.enableServerAudio == 1) {
return <></>
}
return (
<div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1">AudioOutput</div>
@ -80,20 +90,52 @@ export const AudioOutputRow = (_props: AudioOutputRowProps) => {
})
}
</select>
<audio hidden id={AUDIO_ELEMENT_FOR_PLAY_RESULT}></audio>
</div>
</div>
)
}, [guiState.outputAudioDeviceInfo, guiState.audioOutputForGUI])
}, [appState.serverSetting.serverSetting.enableServerAudio, guiState.outputAudioDeviceInfo, guiState.audioOutputForGUI])
useEffect(() => {
setAudioOutputElementId(AUDIO_ELEMENT_FOR_PLAY_RESULT)
}, [initializedRef.current])
const serverAudioOutputRow = useMemo(() => {
if (appState.serverSetting.serverSetting.enableServerAudio == 0) {
return <></>
}
const devices = appState.serverSetting.serverSetting.serverAudioOutputDevices
const hostAPIs = new Set(devices.map(x => { return x.hostAPI }))
const hostAPIOptions = Array.from(hostAPIs).map((x, index) => { return <option value={x} key={index} >{x}</option> })
const filteredDevice = devices.filter(x => { return x.hostAPI == hostApi || hostApi == "" }).map((x, index) => { return <option value={x.index} key={index}>{x.name}</option> })
return (
<div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1">AudioOutput</div>
<div className="body-select-container">
<div className="body-select-container">
<select name="kinds" id="kinds" value={hostApi} onChange={(e) => { setHostApi(e.target.value) }}>
{hostAPIOptions}
</select>
<select className="body-select" value={appState.serverSetting.serverSetting.serverOutputDeviceId} onChange={(e) => {
appState.serverSetting.updateServerSettings({ ...appState.serverSetting.serverSetting, serverOutputDeviceId: Number(e.target.value) })
}}>
{filteredDevice}
</select>
</div>
</div>
</div>
)
}, [hostApi, appState.serverSetting.serverSetting, appState.serverSetting.updateServerSettings])
return (
<>
{audioOutputRow}
<AudioOutputRecordRow />
{appState.serverSetting.serverSetting.enableServerAudio == 0 ? <AudioOutputRecordRow /> : <></>}
{serverAudioOutputRow}
<audio hidden id={AUDIO_ELEMENT_FOR_PLAY_RESULT}></audio>
</>
)
}

50
client/demo/src/components/demo/components/410_AudioDeviceModeRow.tsx Normal file
View File

@ -0,0 +1,50 @@
import React, { useMemo } from "react"
import { useAppState } from "../../../001_provider/001_AppStateProvider"
import { useGuiState } from "../001_GuiStateProvider"
export type AudioDeviceModeRowProps = {
}
export const AudioDeviceModeRow = (_props: AudioDeviceModeRowProps) => {
const appState = useAppState()
const guiState = useGuiState()
const serverAudioInputRow = useMemo(() => {
const enableServerAudio = appState.serverSetting.serverSetting.enableServerAudio
const serverChecked = enableServerAudio == 1 ? true : false
const clientChecked = enableServerAudio == 1 ? false : true
const onDeviceModeChanged = (val: number) => {
if (guiState.isConverting) {
alert("cannot change mode when voice conversion is enabled")
return
}
appState.serverSetting.updateServerSettings({ ...appState.serverSetting.serverSetting, enableServerAudio: val })
}
return (
<div className="body-row split-3-3-4 left-padding-1 guided">
<div className="body-item-title left-padding-1">Device Mode</div>
<div className="body-input-container">
<div className="left-padding-1">
<input type="radio" id="client-device" name="device-mode" checked={clientChecked} onChange={() => { onDeviceModeChanged(0) }} />
<label htmlFor="client-device">client device</label>
</div>
<div className="left-padding-1">
<input className="left-padding-1" type="radio" id="server-device" name="device-mode" checked={serverChecked} onChange={() => { onDeviceModeChanged(1) }} />
<label htmlFor="server-device">server device</label>
</div>
</div>
<div></div>
</div>
)
}, [appState.serverSetting.serverSetting, appState.serverSetting.updateServerSettings, guiState.isConverting])
return (
<>
{serverAudioInputRow}
</>
)
}

56
client/demo/src/components/demo/components/411_IOBufferRow.tsx Normal file
View File

@ -0,0 +1,56 @@
import React, { useMemo } from "react"
import { useAppState } from "../../../001_provider/001_AppStateProvider"
export type IOBufferRowProps = {
}
export const IOBufferRow = (_props: IOBufferRowProps) => {
const appState = useAppState()
const ioBufferRow = useMemo(() => {
if (appState.serverSetting.serverSetting.enableServerAudio == 0) {
return <></>
}
const readBuf = appState.serverSetting.serverSetting.serverInputAudioBufferSize
const writeBuf = appState.serverSetting.serverSetting.serverOutputAudioBufferSize
return (
<div className="body-row split-3-3-4 left-padding-1 guided">
<div className="body-item-title left-padding-1">I/O Buffer</div>
<div className="body-input-container">
<div className="left-padding-1">
In:
<select className="body-select" value={readBuf} onChange={(e) => {
appState.serverSetting.updateServerSettings({ ...appState.serverSetting.serverSetting, serverInputAudioBufferSize: Number(e.target.value) })
appState.workletNodeSetting.trancateBuffer()
}}>
{
[1024 * 4, 1024 * 8, 1024 * 12, 1024 * 16, 1024 * 24, 1024 * 32].map(x => {
return <option key={x} value={x}>{x}</option>
})
}
</select>
</div>
<div className="left-padding-1">
Out:
<select className="body-select" value={writeBuf} onChange={(e) => {
appState.serverSetting.updateServerSettings({ ...appState.serverSetting.serverSetting, serverOutputAudioBufferSize: Number(e.target.value) })
appState.workletNodeSetting.trancateBuffer()
}}>
{
[1024 * 4, 1024 * 8, 1024 * 12, 1024 * 16, 1024 * 24, 1024 * 32].map(x => {
return <option key={x} value={x}>{x}</option>
})
}
</select>
</div>
</div>
<div className="body-item-text"></div>
</div>
)
}, [appState.serverSetting.serverSetting, appState.serverSetting.updateServerSettings])
return ioBufferRow
}

4
client/demo/src/components/demo/components/701_InputChunkNumRow.tsx
View File

@ -6,6 +6,7 @@ export type InputChunkNumRowProps = {
}
export const InputChunkNumRow = (props: InputChunkNumRowProps) => {
const appState = useAppState()
const inputChunkNumRow = useMemo(() => {
let nums: number[]
if (!props.nums) {
@ -20,6 +21,7 @@ export const InputChunkNumRow = (props: InputChunkNumRowProps) => {
<select className="body-select" value={appState.workletNodeSetting.workletNodeSetting.inputChunkNum} onChange={(e) => {
appState.workletNodeSetting.updateWorkletNodeSetting({ ...appState.workletNodeSetting.workletNodeSetting, inputChunkNum: Number(e.target.value) })
appState.workletNodeSetting.trancateBuffer()
appState.serverSetting.updateServerSettings({ ...appState.serverSetting.serverSetting, serverReadChunkSize: Number(e.target.value) })
}}>
{
nums.map(x => {
@ -35,7 +37,7 @@ export const InputChunkNumRow = (props: InputChunkNumRowProps) => {
</div>
)
}, [appState.workletNodeSetting.workletNodeSetting, appState.workletNodeSetting.updateWorkletNodeSetting])
}, [appState.workletNodeSetting.workletNodeSetting, appState.workletNodeSetting.updateWorkletNodeSetting, appState.serverSetting.serverSetting, appState.serverSetting.updateServerSettings])
return inputChunkNumRow
}

146
client/lib/package-lock.json generated
View File

@ -1,16 +1,16 @@
{
"name": "@dannadori/voice-changer-client-js",
"version": "1.0.122",
"version": "1.0.123",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@dannadori/voice-changer-client-js",
"version": "1.0.122",
"version": "1.0.123",
"license": "ISC",
"dependencies": {
"@types/readable-stream": "^2.3.15",
"amazon-chime-sdk-js": "^3.13.0",
"amazon-chime-sdk-js": "^3.14.0",
"buffer": "^6.0.3",
"localforage": "^1.10.0",
"react": "^18.2.0",
@ -19,10 +19,10 @@
},
"devDependencies": {
"@types/audioworklet": "^0.0.45",
"@types/node": "^18.16.3",
"@types/react": "18.2.5",
"@types/react-dom": "18.2.3",
"eslint": "^8.39.0",
"@types/node": "^20.1.0",
"@types/react": "18.2.6",
"@types/react-dom": "18.2.4",
"eslint": "^8.40.0",
"eslint-config-prettier": "^8.8.0",
"eslint-plugin-prettier": "^4.2.1",
"eslint-plugin-react": "^7.32.2",
@ -1428,14 +1428,14 @@
}
},
"node_modules/@eslint/eslintrc": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.0.2.tgz",
"integrity": "sha512-3W4f5tDUra+pA+FzgugqL2pRimUTDJWKr7BINqOpkZrC0uYI0NIc0/JFgBROCU07HR6GieA5m3/rsPIhDmCXTQ==",
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.0.3.tgz",
"integrity": "sha512-+5gy6OQfk+xx3q0d6jGZZC3f3KzAkXc/IanVxd1is/VIIziRqqt3ongQz0FiTUXqTk0c7aDB3OaFuKnuSoJicQ==",
"dev": true,
"dependencies": {
"ajv": "^6.12.4",
"debug": "^4.3.2",
"espree": "^9.5.1",
"espree": "^9.5.2",
"globals": "^13.19.0",
"ignore": "^5.2.0",
"import-fresh": "^3.2.1",
@ -1451,9 +1451,9 @@
}
},
"node_modules/@eslint/js": {
"version": "8.39.0",
"resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.39.0.tgz",
"integrity": "sha512-kf9RB0Fg7NZfap83B3QOqOGg9QmD9yBudqQXzzOtn3i4y7ZUXe5ONeW34Gwi+TxhH4mvj72R1Zc300KUMa9Bng==",
"version": "8.40.0",
"resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.40.0.tgz",
"integrity": "sha512-ElyB54bJIhXQYVKjDSvCkPO1iU1tSAeVQJbllWJq1XQSmmA4dgFk8CbiBGpiOPxleE48vDogxCtmMYku4HSVLA==",
"dev": true,
"engines": {
"node": "^12.22.0 || ^14.17.0 || >=16.0.0"
@ -1829,9 +1829,9 @@
"dev": true
},
"node_modules/@types/node": {
"version": "18.16.3",
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.16.3.tgz",
"integrity": "sha512-OPs5WnnT1xkCBiuQrZA4+YAV4HEJejmHneyraIaxsbev5yCEr6KMwINNFP9wQeFIw8FWcoTqF3vQsa5CDaI+8Q=="
"version": "20.1.0",
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.1.0.tgz",
"integrity": "sha512-O+z53uwx64xY7D6roOi4+jApDGFg0qn6WHcxe5QeqjMaTezBO/mxdfFXIVAVVyNWKx84OmPB3L8kbVYOTeN34A=="
},
"node_modules/@types/prop-types": {
"version": "15.7.5",
@ -1852,9 +1852,9 @@
"dev": true
},
"node_modules/@types/react": {
"version": "18.2.5",
"resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.5.tgz",
"integrity": "sha512-RuoMedzJ5AOh23Dvws13LU9jpZHIc/k90AgmK7CecAYeWmSr3553L4u5rk4sWAPBuQosfT7HmTfG4Rg5o4nGEA==",
"version": "18.2.6",
"resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.6.tgz",
"integrity": "sha512-wRZClXn//zxCFW+ye/D2qY65UsYP1Fpex2YXorHc8awoNamkMZSvBxwxdYVInsHOZZd2Ppq8isnSzJL5Mpf8OA==",
"dev": true,
"dependencies": {
"@types/prop-types": "*",
@ -1863,9 +1863,9 @@
}
},
"node_modules/@types/react-dom": {
"version": "18.2.3",
"resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.2.3.tgz",
"integrity": "sha512-hxXEXWxFJXbY0LMj/T69mznqOZJXNtQMqVxIiirVAZnnpeYiD4zt+lPsgcr/cfWg2VLsxZ1y26vigG03prYB+Q==",
"version": "18.2.4",
"resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.2.4.tgz",
"integrity": "sha512-G2mHoTMTL4yoydITgOGwWdWMVd8sNgyEP85xVmMKAPUBwQWm9wBPQUmvbeF4V3WBY1P7mmL4BkjQ0SqUpf1snw==",
"dev": true,
"dependencies": {
"@types/react": "*"
@ -2259,9 +2259,9 @@
}
},
"node_modules/amazon-chime-sdk-js": {
"version": "3.13.0",
"resolved": "https://registry.npmjs.org/amazon-chime-sdk-js/-/amazon-chime-sdk-js-3.13.0.tgz",
"integrity": "sha512-gRZ8SJgN2+0UMBRhUhAvc06ZklwDyYMEd/xJ17/lORX/XprcTVtRqicEJHxLtgzs61FKmmx1uBz0eGgrn9Dm3A==",
"version": "3.14.0",
"resolved": "https://registry.npmjs.org/amazon-chime-sdk-js/-/amazon-chime-sdk-js-3.14.0.tgz",
"integrity": "sha512-uNwCYygJahvncioPEzg4cDMPbYjpzbu1D92nUeYhUQcDK+qZn8zdPX8BRHI+eUSFBgI47udFPC933ZL79wXqJA==",
"dependencies": {
"@aws-crypto/sha256-js": "^2.0.1",
"@aws-sdk/client-chime-sdk-messaging": "^3.0.0",
@ -3230,15 +3230,15 @@
}
},
"node_modules/eslint": {
"version": "8.39.0",
"resolved": "https://registry.npmjs.org/eslint/-/eslint-8.39.0.tgz",
"integrity": "sha512-mwiok6cy7KTW7rBpo05k6+p4YVZByLNjAZ/ACB9DRCu4YDRwjXI01tWHp6KAUWelsBetTxKK/2sHB0vdS8Z2Og==",
"version": "8.40.0",
"resolved": "https://registry.npmjs.org/eslint/-/eslint-8.40.0.tgz",
"integrity": "sha512-bvR+TsP9EHL3TqNtj9sCNJVAFK3fBN8Q7g5waghxyRsPLIMwL73XSKnZFK0hk/O2ANC+iAoq6PWMQ+IfBAJIiQ==",
"dev": true,
"dependencies": {
"@eslint-community/eslint-utils": "^4.2.0",
"@eslint-community/regexpp": "^4.4.0",
"@eslint/eslintrc": "^2.0.2",
"@eslint/js": "8.39.0",
"@eslint/eslintrc": "^2.0.3",
"@eslint/js": "8.40.0",
"@humanwhocodes/config-array": "^0.11.8",
"@humanwhocodes/module-importer": "^1.0.1",
"@nodelib/fs.walk": "^1.2.8",
@ -3249,8 +3249,8 @@
"doctrine": "^3.0.0",
"escape-string-regexp": "^4.0.0",
"eslint-scope": "^7.2.0",
"eslint-visitor-keys": "^3.4.0",
"espree": "^9.5.1",
"eslint-visitor-keys": "^3.4.1",
"espree": "^9.5.2",
"esquery": "^1.4.2",
"esutils": "^2.0.2",
"fast-deep-equal": "^3.1.3",
@ -3377,9 +3377,9 @@
}
},
"node_modules/eslint-visitor-keys": {
"version": "3.4.0",
"resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.0.tgz",
"integrity": "sha512-HPpKPUBQcAsZOsHAFwTtIKcYlCje62XB7SEAcxjtmW6TD1WVpkS6i6/hOVtTZIl4zGj/mBqpFVGvaDneik+VoQ==",
"version": "3.4.1",
"resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.1.tgz",
"integrity": "sha512-pZnmmLwYzf+kWaM/Qgrvpen51upAktaaiI01nsJD/Yr3lMOdNtq0cxkrrg16w64VtisN6okbs7Q8AfGqj4c9fA==",
"dev": true,
"engines": {
"node": "^12.22.0 || ^14.17.0 || >=16.0.0"
@ -3413,14 +3413,14 @@
}
},
"node_modules/espree": {
"version": "9.5.1",
"resolved": "https://registry.npmjs.org/espree/-/espree-9.5.1.tgz",
"integrity": "sha512-5yxtHSZXRSW5pvv3hAlXM5+/Oswi1AUFqBmbibKb5s6bp3rGIDkyXU6xCoyuuLhijr4SFwPrXRoZjz0AZDN9tg==",
"version": "9.5.2",
"resolved": "https://registry.npmjs.org/espree/-/espree-9.5.2.tgz",
"integrity": "sha512-7OASN1Wma5fum5SrNhFMAMJxOUAbhyfQ8dQ//PJaJbNw0URTPWqIghHWt1MmAANKhHZIYOHruW4Kw4ruUWOdGw==",
"dev": true,
"dependencies": {
"acorn": "^8.8.0",
"acorn-jsx": "^5.3.2",
"eslint-visitor-keys": "^3.4.0"
"eslint-visitor-keys": "^3.4.1"
},
"engines": {
"node": "^12.22.0 || ^14.17.0 || >=16.0.0"
@ -9118,14 +9118,14 @@
"dev": true
},
"@eslint/eslintrc": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.0.2.tgz",
"integrity": "sha512-3W4f5tDUra+pA+FzgugqL2pRimUTDJWKr7BINqOpkZrC0uYI0NIc0/JFgBROCU07HR6GieA5m3/rsPIhDmCXTQ==",
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.0.3.tgz",
"integrity": "sha512-+5gy6OQfk+xx3q0d6jGZZC3f3KzAkXc/IanVxd1is/VIIziRqqt3ongQz0FiTUXqTk0c7aDB3OaFuKnuSoJicQ==",
"dev": true,
"requires": {
"ajv": "^6.12.4",
"debug": "^4.3.2",
"espree": "^9.5.1",
"espree": "^9.5.2",
"globals": "^13.19.0",
"ignore": "^5.2.0",
"import-fresh": "^3.2.1",
@ -9135,9 +9135,9 @@
}
},
"@eslint/js": {
"version": "8.39.0",
"resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.39.0.tgz",
"integrity": "sha512-kf9RB0Fg7NZfap83B3QOqOGg9QmD9yBudqQXzzOtn3i4y7ZUXe5ONeW34Gwi+TxhH4mvj72R1Zc300KUMa9Bng==",
"version": "8.40.0",
"resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.40.0.tgz",
"integrity": "sha512-ElyB54bJIhXQYVKjDSvCkPO1iU1tSAeVQJbllWJq1XQSmmA4dgFk8CbiBGpiOPxleE48vDogxCtmMYku4HSVLA==",
"dev": true
},
"@humanwhocodes/config-array": {
@ -9476,9 +9476,9 @@
"dev": true
},
"@types/node": {
"version": "18.16.3",
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.16.3.tgz",
"integrity": "sha512-OPs5WnnT1xkCBiuQrZA4+YAV4HEJejmHneyraIaxsbev5yCEr6KMwINNFP9wQeFIw8FWcoTqF3vQsa5CDaI+8Q=="
"version": "20.1.0",
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.1.0.tgz",
"integrity": "sha512-O+z53uwx64xY7D6roOi4+jApDGFg0qn6WHcxe5QeqjMaTezBO/mxdfFXIVAVVyNWKx84OmPB3L8kbVYOTeN34A=="
},
"@types/prop-types": {
"version": "15.7.5",
@ -9499,9 +9499,9 @@
"dev": true
},
"@types/react": {
"version": "18.2.5",
"resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.5.tgz",
"integrity": "sha512-RuoMedzJ5AOh23Dvws13LU9jpZHIc/k90AgmK7CecAYeWmSr3553L4u5rk4sWAPBuQosfT7HmTfG4Rg5o4nGEA==",
"version": "18.2.6",
"resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.6.tgz",
"integrity": "sha512-wRZClXn//zxCFW+ye/D2qY65UsYP1Fpex2YXorHc8awoNamkMZSvBxwxdYVInsHOZZd2Ppq8isnSzJL5Mpf8OA==",
"dev": true,
"requires": {
"@types/prop-types": "*",
@ -9510,9 +9510,9 @@
}
},
"@types/react-dom": {
"version": "18.2.3",
"resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.2.3.tgz",
"integrity": "sha512-hxXEXWxFJXbY0LMj/T69mznqOZJXNtQMqVxIiirVAZnnpeYiD4zt+lPsgcr/cfWg2VLsxZ1y26vigG03prYB+Q==",
"version": "18.2.4",
"resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.2.4.tgz",
"integrity": "sha512-G2mHoTMTL4yoydITgOGwWdWMVd8sNgyEP85xVmMKAPUBwQWm9wBPQUmvbeF4V3WBY1P7mmL4BkjQ0SqUpf1snw==",
"dev": true,
"requires": {
"@types/react": "*"
@ -9854,9 +9854,9 @@
"requires": {}
},
"amazon-chime-sdk-js": {
"version": "3.13.0",
"resolved": "https://registry.npmjs.org/amazon-chime-sdk-js/-/amazon-chime-sdk-js-3.13.0.tgz",
"integrity": "sha512-gRZ8SJgN2+0UMBRhUhAvc06ZklwDyYMEd/xJ17/lORX/XprcTVtRqicEJHxLtgzs61FKmmx1uBz0eGgrn9Dm3A==",
"version": "3.14.0",
"resolved": "https://registry.npmjs.org/amazon-chime-sdk-js/-/amazon-chime-sdk-js-3.14.0.tgz",
"integrity": "sha512-uNwCYygJahvncioPEzg4cDMPbYjpzbu1D92nUeYhUQcDK+qZn8zdPX8BRHI+eUSFBgI47udFPC933ZL79wXqJA==",
"requires": {
"@aws-crypto/sha256-js": "^2.0.1",
"@aws-sdk/client-chime-sdk-messaging": "^3.0.0",
@ -10566,15 +10566,15 @@
"dev": true
},
"eslint": {
"version": "8.39.0",
"resolved": "https://registry.npmjs.org/eslint/-/eslint-8.39.0.tgz",
"integrity": "sha512-mwiok6cy7KTW7rBpo05k6+p4YVZByLNjAZ/ACB9DRCu4YDRwjXI01tWHp6KAUWelsBetTxKK/2sHB0vdS8Z2Og==",
"version": "8.40.0",
"resolved": "https://registry.npmjs.org/eslint/-/eslint-8.40.0.tgz",
"integrity": "sha512-bvR+TsP9EHL3TqNtj9sCNJVAFK3fBN8Q7g5waghxyRsPLIMwL73XSKnZFK0hk/O2ANC+iAoq6PWMQ+IfBAJIiQ==",
"dev": true,
"requires": {
"@eslint-community/eslint-utils": "^4.2.0",
"@eslint-community/regexpp": "^4.4.0",
"@eslint/eslintrc": "^2.0.2",
"@eslint/js": "8.39.0",
"@eslint/eslintrc": "^2.0.3",
"@eslint/js": "8.40.0",
"@humanwhocodes/config-array": "^0.11.8",
"@humanwhocodes/module-importer": "^1.0.1",
"@nodelib/fs.walk": "^1.2.8",
@ -10585,8 +10585,8 @@
"doctrine": "^3.0.0",
"escape-string-regexp": "^4.0.0",
"eslint-scope": "^7.2.0",
"eslint-visitor-keys": "^3.4.0",
"espree": "^9.5.1",
"eslint-visitor-keys": "^3.4.1",
"espree": "^9.5.2",
"esquery": "^1.4.2",
"esutils": "^2.0.2",
"fast-deep-equal": "^3.1.3",
@ -10674,9 +10674,9 @@
}
},
"eslint-visitor-keys": {
"version": "3.4.0",
"resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.0.tgz",
"integrity": "sha512-HPpKPUBQcAsZOsHAFwTtIKcYlCje62XB7SEAcxjtmW6TD1WVpkS6i6/hOVtTZIl4zGj/mBqpFVGvaDneik+VoQ==",
"version": "3.4.1",
"resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.1.tgz",
"integrity": "sha512-pZnmmLwYzf+kWaM/Qgrvpen51upAktaaiI01nsJD/Yr3lMOdNtq0cxkrrg16w64VtisN6okbs7Q8AfGqj4c9fA==",
"dev": true
},
"eslint-webpack-plugin": {
@ -10693,14 +10693,14 @@
}
},
"espree": {
"version": "9.5.1",
"resolved": "https://registry.npmjs.org/espree/-/espree-9.5.1.tgz",
"integrity": "sha512-5yxtHSZXRSW5pvv3hAlXM5+/Oswi1AUFqBmbibKb5s6bp3rGIDkyXU6xCoyuuLhijr4SFwPrXRoZjz0AZDN9tg==",
"version": "9.5.2",
"resolved": "https://registry.npmjs.org/espree/-/espree-9.5.2.tgz",
"integrity": "sha512-7OASN1Wma5fum5SrNhFMAMJxOUAbhyfQ8dQ//PJaJbNw0URTPWqIghHWt1MmAANKhHZIYOHruW4Kw4ruUWOdGw==",
"dev": true,
"requires": {
"acorn": "^8.8.0",
"acorn-jsx": "^5.3.2",
"eslint-visitor-keys": "^3.4.0"
"eslint-visitor-keys": "^3.4.1"
}
},
"esquery": {

12
client/lib/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@dannadori/voice-changer-client-js",
"version": "1.0.122",
"version": "1.0.123",
"description": "",
"main": "dist/index.js",
"directories": {
@ -27,10 +27,10 @@
"license": "ISC",
"devDependencies": {
"@types/audioworklet": "^0.0.45",
"@types/node": "^18.16.3",
"@types/react": "18.2.5",
"@types/react-dom": "18.2.3",
"eslint": "^8.39.0",
"@types/node": "^20.1.0",
"@types/react": "18.2.6",
"@types/react-dom": "18.2.4",
"eslint": "^8.40.0",
"eslint-config-prettier": "^8.8.0",
"eslint-plugin-prettier": "^4.2.1",
"eslint-plugin-react": "^7.32.2",
@ -47,7 +47,7 @@
},
"dependencies": {
"@types/readable-stream": "^2.3.15",
"amazon-chime-sdk-js": "^3.13.0",
"amazon-chime-sdk-js": "^3.14.0",
"buffer": "^6.0.3",
"localforage": "^1.10.0",
"react": "^18.2.0",

14
client/lib/src/ServerConfigurator.ts
View File

@ -27,6 +27,20 @@ export class ServerConfigurator {
return info
}
getPerformance = async () => {
const url = this.serverUrl + "/performance"
const info = await new Promise<number[]>((resolve) => {
const request = new Request(url, {
method: 'GET',
});
fetch(request).then(async (response) => {
const json = await response.json() as number[]
resolve(json)
})
})
return info
}
updateSettings = async (key: ServerSettingKey, val: string) => {
const url = this.serverUrl + "/update_settings"
const info = await new Promise<ServerInfo>(async (resolve) => {

3
client/lib/src/VoiceChangerClient.ts
View File

@ -331,6 +331,9 @@ export class VoiceChangerClient {
getServerSettings = () => {
return this.configurator.getSettings()
}
getPerformance = () => {
return this.configurator.getPerformance()
}
getSocketId = () => {
return this.vcInNode.getSocketId()

1
client/lib/src/VoiceChangerWorkletNode.ts
View File

@ -78,6 +78,7 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
console.log(`[SIO] ${this.socket?.id}`)
});
this.socket.on('response', (response: any[]) => {
console.log("response:", response)
const cur = Date.now()
const responseTime = cur - response[0]
const result = response[1] as ArrayBuffer

44
client/lib/src/const.ts
View File

@ -82,6 +82,16 @@ export const ServerSettingKey = {
"f0Detector": "f0Detector",
"recordIO": "recordIO",
"enableServerAudio": "enableServerAudio",
"serverAudioStated": "serverAudioStated",
"serverInputAudioSampleRate": "serverInputAudioSampleRate",
"serverOutputAudioSampleRate": "serverOutputAudioSampleRate",
"serverInputAudioBufferSize": "serverInputAudioBufferSize",
"serverOutputAudioBufferSize": "serverOutputAudioBufferSize",
"serverInputDeviceId": "serverInputDeviceId",
"serverOutputDeviceId": "serverOutputDeviceId",
"serverReadChunkSize": "serverReadChunkSize",
"tran": "tran",
"noiseScale": "noiseScale",
"predictF0": "predictF0",
@ -121,6 +131,17 @@ export type VoiceChangerServerSetting = {
f0Detector: F0Detector // dio or harvest
recordIO: number // 0:off, 1:on
enableServerAudio: number // 0:off, 1:on
serverAudioStated: number // 0:off, 1:on
serverInputAudioSampleRate: number
serverOutputAudioSampleRate: number
serverInputAudioBufferSize: number
serverOutputAudioBufferSize: number
serverInputDeviceId: number
serverOutputDeviceId: number
serverReadChunkSize: number
tran: number // so-vits-svc
noiseScale: number // so-vits-svc
predictF0: number // so-vits-svc
@ -156,6 +177,13 @@ type ModelSlot = {
deprecated: boolean
}
type ServerAudioDevice = {
kind: "audioinput" | "audiooutput",
index: number,
name: string
hostAPI: string
}
export type ServerInfo = VoiceChangerServerSetting & {
status: string
configFile: string,
@ -163,6 +191,9 @@ export type ServerInfo = VoiceChangerServerSetting & {
onnxModelFile: string,
onnxExecutionProviders: OnnxExecutionProvider[]
modelSlots: ModelSlot[]
serverAudioInputDevices: ServerAudioDevice[]
serverAudioOutputDevices: ServerAudioDevice[]
}
export type ServerInfoSoVitsSVC = ServerInfo & {
@ -179,6 +210,15 @@ export const DefaultServerSetting: ServerInfo = {
recordIO: 0,
enableServerAudio: 0,
serverAudioStated: 0,
serverInputAudioSampleRate: 48000,
serverOutputAudioSampleRate: 48000,
serverInputAudioBufferSize: 1024 * 24,
serverOutputAudioBufferSize: 1024 * 24,
serverInputDeviceId: -1,
serverOutputDeviceId: -1,
serverReadChunkSize: 256,
// VC Specific
srcId: 0,
@ -214,7 +254,9 @@ export const DefaultServerSetting: ServerInfo = {
pyTorchModelFile: "",
onnxModelFile: "",
onnxExecutionProviders: [],
modelSlots: []
modelSlots: [],
serverAudioInputDevices: [],
serverAudioOutputDevices: []
}
export const DefaultServerSetting_MMVCv15: ServerInfo = {
...DefaultServerSetting, dstId: 101,

28
client/lib/src/hooks/useClient.ts
View File

@ -25,7 +25,7 @@ export type ClientState = {
bufferingTime: number;
volume: number;
performance: PerformanceData
updatePerformance: (() => Promise<void>) | null
// setClientType: (val: ClientType) => void
// 情報取得
@ -78,6 +78,31 @@ export const useClient = (props: UseClientProps): ClientState => {
const [performance, setPerformance] = useState<PerformanceData>(InitialPerformanceData)
const [volume, setVolume] = useState<number>(0)
//// Server Audio Deviceを使うとき、モニタリングデータはpolling
const updatePerformance = useMemo(() => {
if (!voiceChangerClientRef.current) {
return null
}
return async () => {
if (voiceChangerClientRef.current) {
const performance = await voiceChangerClientRef.current!.getPerformance()
const responseTime = performance[0]
const preprocessTime = performance[1]
const mainprocessTime = performance[2]
const postprocessTime = performance[3]
setPerformance({ responseTime, preprocessTime, mainprocessTime, postprocessTime })
} else {
const responseTime = 0
const preprocessTime = 0
const mainprocessTime = 0
const postprocessTime = 0
setPerformance({ responseTime, preprocessTime, mainprocessTime, postprocessTime })
}
}
}, [voiceChangerClientRef.current])
// (1-4) エラーステータス
const errorCountRef = useRef<number>(0)
@ -168,6 +193,7 @@ export const useClient = (props: UseClientProps): ClientState => {
bufferingTime,
volume,
performance,
updatePerformance,
// setClientType,

6
client/lib/src/hooks/useServerSetting.ts
View File

@ -107,7 +107,11 @@ export const useServerSetting = (props: UseServerSettingProps): ServerSettingSta
const cachedServerSetting = await getItem(INDEXEDDB_KEY_SERVER)
let initialSetting: ServerInfo
if (cachedServerSetting) {
initialSetting = { ...defaultServerSetting, ...cachedServerSetting as ServerInfo, inputSampleRate: 48000 }// sample rateは時限措置
initialSetting = {
...defaultServerSetting, ...cachedServerSetting as ServerInfo,
serverAudioStated: 0,
inputSampleRate: 48000
}// sample rateは時限措置
} else {
initialSetting = { ...defaultServerSetting }
}

5
server/const.py
View File

@ -90,3 +90,8 @@ class EnumPitchExtractorTypes(Enum):
class EnumFrameworkTypes(Enum):
pyTorch = "pyTorch"
onnx = "onnx"
class ServerAudioDeviceTypes(Enum):
audioinput = "audioinput"
audiooutput = "audiooutput"

6
server/restapi/MMVC_Rest_Fileuploader.py
View File

@ -23,6 +23,7 @@ class MMVC_Rest_Fileuploader:
self.voiceChangerManager = voiceChangerManager
self.router = APIRouter()
self.router.add_api_route("/info", self.get_info, methods=["GET"])
self.router.add_api_route("/performance", self.get_performance, methods=["GET"])
self.router.add_api_route(
"/upload_file", self.post_upload_file, methods=["POST"]
)
@ -60,6 +61,11 @@ class MMVC_Rest_Fileuploader:
json_compatible_item_data = jsonable_encoder(info)
return JSONResponse(content=json_compatible_item_data)
def get_performance(self):
info = self.voiceChangerManager.get_performance()
json_compatible_item_data = jsonable_encoder(info)
return JSONResponse(content=json_compatible_item_data)
def post_update_settings(
self, key: str = Form(...), val: Union[int, str, float] = Form(...)
):

22
server/sio/MMVC_Namespace.py
View File

@ -6,6 +6,8 @@ from voice_changer.VoiceChangerManager import VoiceChangerManager
class MMVC_Namespace(socketio.AsyncNamespace):
sid: int = 0
def __init__(self, namespace: str, voiceChangerManager: VoiceChangerManager):
super().__init__(namespace)
self.voiceChangerManager = voiceChangerManager
@ -17,24 +19,32 @@ class MMVC_Namespace(socketio.AsyncNamespace):
return cls._instance
def on_connect(self, sid, environ):
print('[{}] connet sid : {}'.format(datetime.now().strftime('%Y-%m-%d %H:%M:%S'), sid))
self.sid = sid
print(
"[{}] connet sid : {}".format(
datetime.now().strftime("%Y-%m-%d %H:%M:%S"), sid
)
)
pass
async def on_request_message(self, sid, msg):
self.sid = sid
timestamp = int(msg[0])
data = msg[1]
if (isinstance(data, str)):
if isinstance(data, str):
print(type(data))
print(data)
await self.emit('response', [timestamp, 0], to=sid)
await self.emit("response", [timestamp, 0], to=sid)
else:
unpackedData = np.array(struct.unpack('<%sh' % (len(data) // struct.calcsize('<h')), data)).astype(np.int16)
unpackedData = np.array(
struct.unpack("<%sh" % (len(data) // struct.calcsize("<h")), data)
).astype(np.int16)
res = self.voiceChangerManager.changeVoice(unpackedData)
audio1 = res[0]
perf = res[1] if len(res) == 2 else [0, 0, 0]
bin = struct.pack('<%sh' % len(audio1), *audio1)
await self.emit('response', [timestamp, bin, perf], to=sid)
bin = struct.pack("<%sh" % len(audio1), *audio1)
await self.emit("response", [timestamp, bin, perf], to=sid)
def on_disconnect(self, sid):
# print('[{}] disconnect'.format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')))

BIN
server/test.wav Normal file
View File

Binary file not shown.

58
server/voice_changer/Local/AudioDeviceList.py Normal file
View File

@ -0,0 +1,58 @@
import pyaudio
# import json
from dataclasses import dataclass
from const import ServerAudioDeviceTypes
@dataclass
class ServerAudioDevice:
kind: ServerAudioDeviceTypes = ServerAudioDeviceTypes.audioinput
index: int = 0
name: str = ""
hostAPI: str = ""
def list_audio_device():
audio = pyaudio.PyAudio()
audio_input_devices: list[ServerAudioDevice] = []
audio_output_devices: list[ServerAudioDevice] = []
# audio_devices = {}
host_apis = []
for api_index in range(audio.get_host_api_count()):
host_apis.append(audio.get_host_api_info_by_index(api_index)["name"])
for x in range(0, audio.get_device_count()):
device = audio.get_device_info_by_index(x)
try:
deviceName = device["name"].encode("shift-jis").decode("utf-8")
except (UnicodeDecodeError, UnicodeEncodeError):
deviceName = device["name"]
deviceIndex = device["index"]
hostAPI = host_apis[device["hostApi"]]
if device["maxInputChannels"] > 0:
audio_input_devices.append(
ServerAudioDevice(
kind=ServerAudioDeviceTypes.audioinput,
index=deviceIndex,
name=deviceName,
hostAPI=hostAPI,
)
)
if device["maxOutputChannels"] > 0:
audio_output_devices.append(
ServerAudioDevice(
kind=ServerAudioDeviceTypes.audiooutput,
index=deviceIndex,
name=deviceName,
hostAPI=hostAPI,
)
)
return audio_input_devices, audio_output_devices

140
server/voice_changer/VoiceChanger.py
View File

@ -1,4 +1,6 @@
from typing import Any, Union, cast
import socketio
from const import TMP_DIR, ModelType
import torch
import os
@ -9,6 +11,7 @@ import resampy
from voice_changer.IORecorder import IORecorder
from voice_changer.Local.AudioDeviceList import ServerAudioDevice, list_audio_device
from voice_changer.utils.LoadModelParams import LoadModelParams
from voice_changer.utils.Timer import Timer
@ -21,6 +24,10 @@ from Exceptions import (
ONNXInputArgumentException,
)
from voice_changer.utils.VoiceChangerParams import VoiceChangerParams
import pyaudio
import threading
import struct
import time
providers = [
"OpenVINOExecutionProvider",
@ -42,10 +49,38 @@ class VoiceChangerSettings:
crossFadeOverlapSize: int = 4096
recordIO: int = 0 # 0:off, 1:on
serverAudioInputDevices: list[ServerAudioDevice] = field(default_factory=lambda: [])
serverAudioOutputDevices: list[ServerAudioDevice] = field(
default_factory=lambda: []
)
enableServerAudio: int = 0 # 0:off, 1:on
serverAudioStated: int = 0 # 0:off, 1:on
serverInputAudioSampleRate: int = 48000
serverOutputAudioSampleRate: int = 48000
serverInputAudioBufferSize: int = 1024 * 24
serverOutputAudioBufferSize: int = 1024 * 24
serverInputDeviceId: int = -1
serverOutputDeviceId: int = -1
serverReadChunkSize: int = 256
performance: list[int] = field(default_factory=lambda: [0, 0, 0, 0])
# ↓mutableな物だけ列挙
intData: list[str] = field(
default_factory=lambda: ["inputSampleRate", "crossFadeOverlapSize", "recordIO"]
default_factory=lambda: [
"inputSampleRate",
"crossFadeOverlapSize",
"recordIO",
"enableServerAudio",
"serverAudioStated",
"serverInputAudioSampleRate",
"serverOutputAudioSampleRate",
"serverInputAudioBufferSize",
"serverOutputAudioBufferSize",
"serverInputDeviceId",
"serverOutputDeviceId",
"serverReadChunkSize",
]
)
floatData: list[str] = field(
default_factory=lambda: ["crossFadeOffsetRate", "crossFadeEndRate"]
@ -53,11 +88,105 @@ class VoiceChangerSettings:
strData: list[str] = field(default_factory=lambda: [])
def serverLocal(_vc):
vc: VoiceChanger = _vc
audio = pyaudio.PyAudio()
def createAudioInput(deviceId: int, sampleRate: int, bufferSize: int):
audio_input_stream = audio.open(
format=pyaudio.paInt16,
channels=1,
rate=sampleRate,
# frames_per_buffer=32768,
frames_per_buffer=bufferSize,
input_device_index=deviceId,
input=True,
)
return audio_input_stream
def createAudioOutput(deviceId: int, sampleRate: int, bufferSize: int):
audio_output_stream = audio.open(
format=pyaudio.paInt16,
channels=1,
rate=sampleRate,
# frames_per_buffer=32768,
frames_per_buffer=bufferSize,
output_device_index=deviceId,
output=True,
)
return audio_output_stream
currentInputDeviceId = -1
currentInputSampleRate = -1
currentInputBufferSize = -1
currentOutputDeviceId = -1
currentOutputSampleRate = -1
currentOutputBufferSize = -1
audio_input_stream = None
audio_output_stream = None
while True:
if (
vc.settings.enableServerAudio == 0
or vc.settings.serverAudioStated == 0
or vc.settings.serverInputDeviceId == -1
or vc.settings.serverOutputDeviceId == -1
):
time.sleep(2)
else:
if (
currentInputDeviceId != vc.settings.serverInputDeviceId
or currentInputSampleRate != vc.settings.serverInputAudioSampleRate
or currentInputBufferSize != vc.settings.serverInputAudioBufferSize
):
currentInputDeviceId = vc.settings.serverInputDeviceId
currentInputSampleRate = vc.settings.serverInputAudioSampleRate
currentInputBufferSize = vc.settings.serverInputAudioBufferSize
if audio_input_stream is not None:
audio_input_stream.close()
audio_input_stream = createAudioInput(
currentInputDeviceId,
currentInputSampleRate,
currentInputBufferSize,
)
if (
currentOutputDeviceId != vc.settings.serverOutputDeviceId
or currentOutputSampleRate != vc.settings.serverOutputAudioSampleRate
or currentOutputBufferSize != vc.settings.serverOutputAudioBufferSize
):
currentOutputDeviceId = vc.settings.serverOutputDeviceId
currentOutputSampleRate = vc.settings.serverOutputAudioSampleRate
currentOutputBufferSize = vc.settings.serverOutputAudioBufferSize
if audio_output_stream is not None:
audio_output_stream.close()
audio_output_stream = createAudioOutput(
currentOutputDeviceId,
currentOutputSampleRate,
currentOutputBufferSize,
)
in_wav = audio_input_stream.read(
vc.settings.serverReadChunkSize * 128, exception_on_overflow=False
)
unpackedData = np.array(
struct.unpack("<%sh" % (len(in_wav) // struct.calcsize("<h")), in_wav)
).astype(np.int16)
with Timer("all_inference_time") as t:
out_wav, times = vc.on_request(unpackedData)
all_inference_time = t.secs
performance = [all_inference_time] + times
performance = [round(x, 2) * 1000 for x in performance]
vc.settings.performance = performance
audio_output_stream.write(out_wav.tobytes())
class VoiceChanger:
settings: VoiceChangerSettings
voiceChanger: VoiceChangerModel
ioRecorder: IORecorder
sola_buffer: AudioInOut
namespace: socketio.AsyncNamespace | None = None
def __init__(self, params: VoiceChangerParams):
# 初期化
@ -78,6 +207,12 @@ class VoiceChanger:
and torch.backends.mps.is_available()
)
audioinput, audiooutput = list_audio_device()
self.settings.serverAudioInputDevices = audioinput
self.settings.serverAudioOutputDevices = audiooutput
thread = threading.Thread(target=serverLocal, args=(self,))
thread.start()
print(
f"VoiceChanger Initialized (GPU_NUM:{self.gpu_num}, mps_enabled:{self.mps_enabled})"
)
@ -140,6 +275,9 @@ class VoiceChanger:
data.update(self.voiceChanger.get_info())
return data
def get_performance(self):
return self.settings.performance
def update_settings(self, key: str, val: Any):
if key in self.settings.intData:
setattr(self.settings, key, int(val))

7
server/voice_changer/VoiceChangerManager.py
View File

@ -33,6 +33,13 @@ class VoiceChangerManager(object):
else:
return {"status": "ERROR", "msg": "no model loaded"}
def get_performance(self):
if hasattr(self, "voiceChanger"):
info = self.voiceChanger.get_performance()
return info
else:
return {"status": "ERROR", "msg": "no model loaded"}
def update_settings(self, key: str, val: str | int | float):
if hasattr(self, "voiceChanger"):
info = self.voiceChanger.update_settings(key, val)